sqlce 0.1.0__tar.gz → 0.1.1__tar.gz

{sqlce-0.1.0 → sqlce-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sqlce
-Version: 0.1.0
+Version: 0.1.1
 Summary: Cross-platform lib for reading SQL Server CE (.sdf)
 Author-Email: boykopovar <boykopovar@gmail.com>
 License: MIT
@@ -27,21 +27,22 @@ Description-Content-Type: text/markdown
  
 Анализ внутреннего устройства формата по декомпилированным компонентам SQL CE. Цель - предоставить кроссплатформенную и независимую от рантайма реализацию чтения `.sdf` и документацию формата.
 
-## Структура
 
 - `core/` - реализация парсера на C++
 - `bindings/` - Биндинги для C++ библиотеки к другим языкам
 - `python/sqlce/` - Python API
 - `research/` - Экспериментальные скрипты
 
-## Установка
 
 ```bash
 pip install sqlce
 ```
 
-[Документирование формата](docs/4.0/ru/00-overview.md)
 
-Текущая [реализация](main.py) поддерживает чтение списка таблиц, структуры таблиц, содержимого записей. Использует c++.
+Текущая [реализация](main.py) поддерживает чтение (поддерживается передача пароля) списка таблиц, структуры таблиц, содержимого записей. Использует c++.
+
+[Документация](docs/ru/python.md)
+
+[Documentation](docs/en/python.md)
 
 [Соглашения по написанию кода](docs/CONVENTIONS.md)

{sqlce-0.1.0 → sqlce-0.1.1}/README.md

@@ -4,21 +4,22 @@
  
 Анализ внутреннего устройства формата по декомпилированным компонентам SQL CE. Цель - предоставить кроссплатформенную и независимую от рантайма реализацию чтения `.sdf` и документацию формата.
 
-## Структура
 
 - `core/` - реализация парсера на C++
 - `bindings/` - Биндинги для C++ библиотеки к другим языкам
 - `python/sqlce/` - Python API
 - `research/` - Экспериментальные скрипты
 
-## Установка
 
 ```bash
 pip install sqlce
 ```
 
-[Документирование формата](docs/4.0/ru/00-overview.md)
 
-Текущая [реализация](main.py) поддерживает чтение списка таблиц, структуры таблиц, содержимого записей. Использует c++.
+Текущая [реализация](main.py) поддерживает чтение (поддерживается передача пароля) списка таблиц, структуры таблиц, содержимого записей. Использует c++.
+
+[Документация](docs/ru/python.md)
+
+[Documentation](docs/en/python.md)
 
 [Соглашения по написанию кода](docs/CONVENTIONS.md)

{sqlce-0.1.0 → sqlce-0.1.1}/bindings/src/module.cpp

@@ -9,6 +9,7 @@
 
 #include "sdf/application/ColumnSchema.hpp"
 #include "sdf/application/SdfDatabase.hpp"
+#include "sdf/domain/EncryptionMode.hpp"
 #include "sdf/domain/Row.hpp"
 
 #include "value_convert.hpp"
@@ -26,7 +27,9 @@ constexpr char ModuleDoc[] = "Native reader for SQL Server Compact (.sdf) databa
 constexpr char DatabaseClassName[] = "SdfDatabase";
 constexpr char DatabaseDoc[] = "Reads tables, schemas and rows from a .sdf database file.";
 constexpr char InitDoc[] = "Open a .sdf file at the given path.";
+constexpr char InitWithPasswordDoc[] = "Open a password-protected .sdf file, auto-detecting its encryption mode.";
 constexpr char PathArgName[] = "path";
+constexpr char PasswordArgName[] = "password";
 constexpr char ListTablesName[] = "list_tables";
 constexpr char ListTablesDoc[] = "Return the names of all tables in the database.";
 constexpr char TableSchemaName[] = "table_schema";
@@ -44,6 +47,9 @@ constexpr char DeclaredSizeAttrName[] = "declared_size";
 constexpr char PrecisionAttrName[] = "precision";
 constexpr char ScaleAttrName[] = "scale";
 
+constexpr char UnsupportedEncryptionModeErrorName[] = "UnsupportedEncryptionModeError";
+constexpr char InvalidPasswordErrorName[] = "InvalidPasswordError";
+
 py::dict RowToDict(const domain::Row& row)
 {
     py::dict result;
@@ -75,12 +81,40 @@ std::optional<int> OptionalByteToInt(std::optional<std::uint8_t> value)
     return static_cast<int>(*value);
 }
 
+void RegisterExceptions(py::module_& module)
+{
+    static const py::exception<domain::UnsupportedEncryptionModeException> unsupportedModeError(
+        module, UnsupportedEncryptionModeErrorName, PyExc_ValueError);
+    static const py::exception<domain::InvalidPasswordException> invalidPasswordError(
+        module, InvalidPasswordErrorName, PyExc_ValueError);
+
+    py::register_exception_translator([](std::exception_ptr lastException) {
+        try
+        {
+            if (lastException)
+            {
+                std::rethrow_exception(lastException);
+            }
+        }
+        catch (const domain::UnsupportedEncryptionModeException& error)
+        {
+            PyErr_SetString(unsupportedModeError.ptr(), error.what());
+        }
+        catch (const domain::InvalidPasswordException& error)
+        {
+            PyErr_SetString(invalidPasswordError.ptr(), error.what());
+        }
+    });
+}
+
 }
 
 PYBIND11_MODULE(_sdf_native, module)
 {
     module.doc() = ModuleDoc;
 
+    RegisterExceptions(module);
+
     py::class_<application::ColumnSchema>(module, SchemaClassName, SchemaDoc)
         .def_property_readonly(
             OrdinalAttrName,
@@ -103,6 +137,11 @@ PYBIND11_MODULE(_sdf_native, module)
 
     py::class_<application::SdfDatabase>(module, DatabaseClassName, DatabaseDoc)
         .def(py::init<const std::string&>(), py::arg(PathArgName), InitDoc)
+        .def(
+            py::init<const std::string&, const std::string&>(),
+            py::arg(PathArgName),
+            py::arg(PasswordArgName),
+            InitWithPasswordDoc)
         .def(ListTablesName, &application::SdfDatabase::ListTables, ListTablesDoc)
         .def(
             TableSchemaName,

{sqlce-0.1.0 → sqlce-0.1.1}/core/CMakeLists.txt

@@ -18,15 +18,19 @@ add_library(libsdf
         src/domain/NumericValue.cpp
         src/domain/Row.cpp
         src/domain/TableDef.cpp
+        src/infrastructure/AesCbc.cpp
         src/infrastructure/BinaryReader.cpp
         src/infrastructure/FileStorage.cpp
         src/infrastructure/PageView.cpp
+        src/infrastructure/Sha256.cpp
+        src/infrastructure/Sha512.cpp
         src/parsing/CatalogPageScanner.cpp
         src/parsing/CatalogRow.cpp
         src/parsing/CatalogRowDecoder.cpp
         src/parsing/CatalogSchema.cpp
         src/parsing/LobChainRegistry.cpp
         src/parsing/RowDecoder.cpp
+        src/parsing/SdfPageCipher.cpp
         src/parsing/TableCatalogBuilder.cpp
         src/parsing/TextDecoder.cpp
         src/application/SdfDatabase.cpp
@@ -45,7 +49,7 @@ else()
     target_compile_options(libsdf PRIVATE -Wall -Wextra)
 endif()
 
-add_executable(sdf_dump app/main.cpp)
+add_executable(sdf_dump main.cpp)
 target_link_libraries(sdf_dump PRIVATE libsdf)
 
 if(MSVC)

{sqlce-0.1.0 → sqlce-0.1.1}/core/include/sdf/application/SdfDatabase.hpp

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "sdf/application/ColumnSchema.hpp"
+#include "sdf/domain/IPageCipher.hpp"
 #include "sdf/domain/IPageStorage.hpp"
 #include "sdf/domain/Row.hpp"
 #include "sdf/domain/TableDef.hpp"
@@ -22,6 +23,7 @@ class SdfDatabase
 {
 public:
     explicit SdfDatabase(const std::string& path);
+    SdfDatabase(const std::string& path, const std::string& password);
 
     std::vector<std::string> ListTables() const;
     std::vector<ColumnSchema> TableSchema(const std::string& tableName) const;
@@ -36,6 +38,10 @@ private:
 
     std::map<std::string, domain::TableDef> _tables;
 
+    explicit SdfDatabase(std::unique_ptr<domain::IPageStorage> storage);
+
+    static std::unique_ptr<domain::IPageStorage> OpenStorage(const std::string& path, const std::string& password);
+
     void AssignDataPages();
     const domain::TableDef& RequireTable(const std::string& tableName) const;
 };

sqlce-0.1.1/core/include/sdf/domain/EncryptionMode.hpp

@@ -0,0 +1,48 @@
+#ifndef SDF_DOMAIN_ENCRYPTION_MODE_HPP
+#define SDF_DOMAIN_ENCRYPTION_MODE_HPP
+
+#include <cstdint>
+#include <stdexcept>
+#include <string>
+
+namespace sdf::domain
+{
+
+enum class EncryptionMode : std::uint32_t
+{
+    None = 0,
+    Rc4Sha1 = 1,
+    Aes128Sha1 = 2,
+    Aes128Sha256 = 3,
+    Aes256Sha512 = 4
+};
+
+class UnsupportedEncryptionModeException final : public std::runtime_error
+{
+public:
+    explicit UnsupportedEncryptionModeException(EncryptionMode mode)
+        : std::runtime_error("unsupported .sdf encryption mode: " + std::to_string(static_cast<std::uint32_t>(mode)))
+        , _mode(mode)
+    {
+    }
+
+    EncryptionMode Mode() const
+    {
+        return _mode;
+    }
+
+private:
+    EncryptionMode _mode;
+};
+
+class InvalidPasswordException final : public std::runtime_error
+{
+public:
+    InvalidPasswordException() : std::runtime_error("password verification failed")
+    {
+    }
+};
+
+}
+
+#endif

sqlce-0.1.1/core/include/sdf/domain/IPageCipher.hpp

@@ -0,0 +1,23 @@
+#ifndef SDF_DOMAIN_I_PAGE_CIPHER_HPP
+#define SDF_DOMAIN_I_PAGE_CIPHER_HPP
+
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <vector>
+
+namespace sdf::domain
+{
+
+class IPageCipher
+{
+public:
+    virtual ~IPageCipher() = default;
+
+    virtual bool VerifyPassword() const = 0;
+    virtual std::vector<std::uint8_t> DecryptPage(std::size_t pageNumber, std::span<const std::uint8_t> page) const = 0;
+};
+
+}
+
+#endif

{sqlce-0.1.0 → sqlce-0.1.1}/core/include/sdf/domain/PageLayout.hpp

@@ -14,6 +14,14 @@ constexpr std::size_t LobPageHeaderLength = 16;
 constexpr std::size_t SlotArrayBaseIndex = 1017;
 constexpr std::size_t RowHeaderLength = 28;
 
+constexpr std::size_t PlaintextHeaderLength = 16;
+constexpr std::size_t Page0ModeOffset = 184;
+constexpr std::size_t Page0VerifierOffset = 76;
+constexpr std::size_t Page0VerifierLength = 96;
+constexpr std::size_t Page0KeyParamOffset = 188;
+constexpr std::size_t Page0KeyParamLength = 4;
+constexpr std::size_t MaxPasswordLength = 40;
+
 }
 
 #endif

sqlce-0.1.1/core/include/sdf/infrastructure/AesCbc.hpp

@@ -0,0 +1,30 @@
+#ifndef SDF_INFRASTRUCTURE_AES_CBC_HPP
+#define SDF_INFRASTRUCTURE_AES_CBC_HPP
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <vector>
+
+namespace sdf::infrastructure
+{
+
+class AesCbcDecryptor
+{
+public:
+    explicit AesCbcDecryptor(std::span<const std::uint8_t> key, std::span<const std::uint8_t, 16> iv);
+
+    std::vector<std::uint8_t> Decrypt(std::span<const std::uint8_t> ciphertext) const;
+
+private:
+    std::vector<std::uint32_t> _roundKeys;
+    std::size_t _roundCount;
+    std::array<std::uint8_t, 16> _iv;
+
+    void DecryptBlock(const std::uint8_t* in, std::uint8_t* out) const;
+};
+
+}
+
+#endif

{sqlce-0.1.0 → sqlce-0.1.1}/core/include/sdf/infrastructure/FileStorage.hpp

@@ -2,18 +2,23 @@
 #define SDF_INFRASTRUCTURE_FILE_STORAGE_HPP
 
 #include <cstdint>
+#include <optional>
 #include <string>
 #include <vector>
 
+#include "sdf/domain/IPageCipher.hpp"
 #include "sdf/domain/IPageStorage.hpp"
 
 namespace sdf::infrastructure
 {
 
+std::vector<std::uint8_t> ReadFirstPageRaw(const std::string& path);
+
 class FileStorage final : public domain::IPageStorage
 {
 public:
     explicit FileStorage(const std::string& path);
+    FileStorage(const std::string& path, const domain::IPageCipher& cipher);
 
     std::size_t PageCount() const override;
     std::span<const std::uint8_t> PageBytes(std::size_t pageNumber) const override;
@@ -21,6 +26,8 @@ public:
 private:
     std::vector<std::uint8_t> _fileBytes;
     std::size_t _pageCount;
+
+    void Load(const std::string& path);
 };
 
 }

sqlce-0.1.1/core/include/sdf/infrastructure/Sha256.hpp

@@ -0,0 +1,16 @@
+#ifndef SDF_INFRASTRUCTURE_SHA256_HPP
+#define SDF_INFRASTRUCTURE_SHA256_HPP
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+
+namespace sdf::infrastructure
+{
+
+std::array<std::uint8_t, 32> Sha256(std::span<const std::uint8_t> message);
+
+}
+
+#endif

sqlce-0.1.1/core/include/sdf/infrastructure/Sha512.hpp

@@ -0,0 +1,16 @@
+#ifndef SDF_INFRASTRUCTURE_SHA512_HPP
+#define SDF_INFRASTRUCTURE_SHA512_HPP
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+
+namespace sdf::infrastructure
+{
+
+std::array<std::uint8_t, 64> Sha512(std::span<const std::uint8_t> message);
+
+}
+
+#endif

sqlce-0.1.1/core/include/sdf/parsing/SdfPageCipher.hpp

@@ -0,0 +1,41 @@
+#ifndef SDF_PARSING_SDF_PAGE_CIPHER_HPP
+#define SDF_PARSING_SDF_PAGE_CIPHER_HPP
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <string>
+#include <vector>
+
+#include "sdf/domain/EncryptionMode.hpp"
+#include "sdf/domain/IPageCipher.hpp"
+
+namespace sdf::parsing
+{
+
+class SdfPageCipher final : public domain::IPageCipher
+{
+public:
+    SdfPageCipher(std::span<const std::uint8_t> page0, std::string password);
+
+    static domain::EncryptionMode ReadMode(std::span<const std::uint8_t> page0);
+
+    bool VerifyPassword() const override;
+    std::vector<std::uint8_t> DecryptPage(std::size_t pageNumber, std::span<const std::uint8_t> page) const override;
+
+private:
+    domain::EncryptionMode _mode;
+    std::string _password;
+    std::vector<std::uint8_t> _passwordUtf16Le;
+    std::vector<std::uint8_t> _page0Ciphertext;
+    std::array<std::uint8_t, 4> _page0KeyParam;
+
+    std::vector<std::uint8_t> DecryptPage0(std::span<const std::uint8_t> page0) const;
+    std::vector<std::uint8_t> DeriveKey(std::span<const std::uint8_t> keyParam) const;
+    std::vector<std::uint8_t> HashPasswordWith(std::span<const std::uint8_t> keyParam) const;
+};
+
+}
+
+#endif

{sqlce-0.1.0/core/app → sqlce-0.1.1/core}/main.cpp

@@ -1,9 +1,11 @@
 #include <algorithm>
 #include <filesystem>
 #include <iostream>
+#include <memory>
 #include <vector>
 
-#include "sdf/application/SdfDatabase.hpp"
+#include "include/sdf/application/SdfDatabase.hpp"
+#include "include/sdf/domain/EncryptionMode.hpp"
 
 namespace
 {
@@ -71,12 +73,28 @@ void PrintTableRows(const std::vector<std::string>& columnNames, const std::vect
     }
 }
 
-void DumpFile(const std::filesystem::path& sdfPath)
+void DumpFile(const std::filesystem::path& sdfPath, const std::string& password)
 {
     std::cout << "\n" << sdfPath.filename().string() << "\n";
 
-    sdf::application::SdfDatabase db(sdfPath.string());
-    std::vector<std::string> tableNames = db.ListTables();
+    std::unique_ptr<sdf::application::SdfDatabase> db;
+    try
+    {
+        db = password.empty() ? std::make_unique<sdf::application::SdfDatabase>(sdfPath.string())
+                               : std::make_unique<sdf::application::SdfDatabase>(sdfPath.string(), password);
+    }
+    catch (const sdf::domain::UnsupportedEncryptionModeException& error)
+    {
+        std::cout << "skipped: " << error.what() << "\n";
+        return;
+    }
+    catch (const sdf::domain::InvalidPasswordException& error)
+    {
+        std::cout << "skipped: " << error.what() << "\n";
+        return;
+    }
+
+    std::vector<std::string> tableNames = db->ListTables();
 
     if (tableNames.empty())
     {
@@ -88,7 +106,7 @@ void DumpFile(const std::filesystem::path& sdfPath)
     {
         std::cout << "\ntable: " << tableName << "\n";
 
-        for (const sdf::application::ColumnSchema& column : db.TableSchema(tableName))
+        for (const sdf::application::ColumnSchema& column : db->TableSchema(tableName))
         {
             std::cout << "  " << column.ordinal << ". " << column.name << " " << column.typeName
                        << " size=" << column.declaredSize;
@@ -100,7 +118,7 @@ void DumpFile(const std::filesystem::path& sdfPath)
             std::cout << "\n";
         }
 
-        const std::vector<sdf::domain::Row> rows = db.ReadTable(tableName);
+        const std::vector<sdf::domain::Row> rows = db->ReadTable(tableName);
         if (rows.empty())
         {
             std::cout << "  (no rows)\n";
@@ -120,36 +138,58 @@ void DumpFile(const std::filesystem::path& sdfPath)
 
 int main(int argc, char** argv)
 {
-    const std::filesystem::path directory = argc > 1 ? std::filesystem::path(argv[1]) : std::filesystem::path("research/raw/examples");
+    std::filesystem::path directory("research/raw/examples");
+    std::string password;
 
-    std::cout << "current working directory: " << std::filesystem::current_path() << "\n";
-    std::cout << "looking for: " << std::filesystem::absolute(directory) << "\n";
-
-    if (!std::filesystem::is_directory(directory))
+    for (int i = 1; i < argc; ++i)
     {
-        std::cerr << "directory not found: " << directory << "\n";
-        return 1;
+        const std::string argument = argv[i];
+        if (argument == "--password" && i + 1 < argc)
+        {
+            password = argv[++i];
+        }
+        else
+        {
+            directory = std::filesystem::path(argument);
+        }
     }
 
-    std::vector<std::filesystem::path> sdfFiles;
-    for (const std::filesystem::directory_entry& entry : std::filesystem::directory_iterator(directory))
+    std::cout << "current working directory: " << std::filesystem::current_path() << "\n";
+
+    std::vector<std::pair<std::filesystem::path, std::string>> directoriesToScan;
+    directoriesToScan.emplace_back(directory, password);
+    directoriesToScan.emplace_back(directory / "protected", password.empty() ? "secret123" : password);
+
+    for (const auto& [scanDirectory, scanPassword] : directoriesToScan)
     {
-        if (entry.path().extension() == ".sdf")
+        std::cout << "looking for: " << std::filesystem::absolute(scanDirectory) << "\n";
+
+        if (!std::filesystem::is_directory(scanDirectory))
         {
-            sdfFiles.push_back(entry.path());
+            std::cerr << "directory not found: " << scanDirectory << "\n";
+            continue;
         }
-    }
-    std::sort(sdfFiles.begin(), sdfFiles.end());
 
-    if (sdfFiles.empty())
-    {
-        std::cout << "no .sdf files found in " << directory << "\n";
-        return 0;
-    }
+        std::vector<std::filesystem::path> sdfFiles;
+        for (const std::filesystem::directory_entry& entry : std::filesystem::directory_iterator(scanDirectory))
+        {
+            if (entry.path().extension() == ".sdf")
+            {
+                sdfFiles.push_back(entry.path());
+            }
+        }
+        std::sort(sdfFiles.begin(), sdfFiles.end());
 
-    for (const std::filesystem::path& sdfFile : sdfFiles)
-    {
-        DumpFile(sdfFile);
+        if (sdfFiles.empty())
+        {
+            std::cout << "no .sdf files found in " << scanDirectory << "\n";
+            continue;
+        }
+
+        for (const std::filesystem::path& sdfFile : sdfFiles)
+        {
+            DumpFile(sdfFile, scanPassword);
+        }
     }
 
     return 0;

{sqlce-0.1.0 → sqlce-0.1.1}/core/src/application/SdfDatabase.cpp

@@ -8,13 +8,34 @@
 #include "sdf/parsing/CatalogRowDecoder.hpp"
 #include "sdf/parsing/LobChainRegistry.hpp"
 #include "sdf/parsing/RowDecoder.hpp"
+#include "sdf/parsing/SdfPageCipher.hpp"
 #include "sdf/parsing/TableCatalogBuilder.hpp"
 
 namespace sdf::application
 {
 
-SdfDatabase::SdfDatabase(const std::string& path)
-    : _storage(std::make_unique<infrastructure::FileStorage>(path))
+SdfDatabase::SdfDatabase(const std::string& path) : SdfDatabase(OpenStorage(path, std::string()))
+{
+}
+
+SdfDatabase::SdfDatabase(const std::string& path, const std::string& password) : SdfDatabase(OpenStorage(path, password))
+{
+}
+
+std::unique_ptr<domain::IPageStorage> SdfDatabase::OpenStorage(const std::string& path, const std::string& password)
+{
+    if (password.empty())
+    {
+        return std::make_unique<infrastructure::FileStorage>(path);
+    }
+
+    const std::vector<std::uint8_t> firstPage = infrastructure::ReadFirstPageRaw(path);
+    const parsing::SdfPageCipher cipher(firstPage, password);
+    return std::make_unique<infrastructure::FileStorage>(path, cipher);
+}
+
+SdfDatabase::SdfDatabase(std::unique_ptr<domain::IPageStorage> storage)
+    : _storage(std::move(storage))
     , _pageScanner(std::make_shared<parsing::CatalogPageScanner>())
     , _tableCatalogBuilder(
           std::make_shared<parsing::TableCatalogBuilder>(_pageScanner, std::make_shared<parsing::CatalogRowDecoder>()))