classifyre-cli 0.4.45__tar.gz → 0.4.46__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. classifyre_cli-0.4.46/.turbo/turbo-build.log +3 -0
  2. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/PKG-INFO +1 -1
  3. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/package.json +1 -1
  4. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/pyproject.toml +1 -1
  5. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/models/generated_input.py +84 -3
  6. classifyre_cli-0.4.46/src/sources/local_folder/__init__.py +3 -0
  7. classifyre_cli-0.4.46/src/sources/local_folder/source.py +146 -0
  8. classifyre_cli-0.4.46/tests/test_local_folder_source.py +335 -0
  9. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/uv.lock +2 -2
  10. classifyre_cli-0.4.45/.turbo/turbo-build.log +0 -3
  11. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/.gitignore +0 -0
  12. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/.python-version +0 -0
  13. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/README.md +0 -0
  14. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/main.py +0 -0
  15. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/scripts/generate_models.py +0 -0
  16. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/__init__.py +0 -0
  17. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/config.py +0 -0
  18. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/__init__.py +0 -0
  19. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/base.py +0 -0
  20. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/broken_links/__init__.py +0 -0
  21. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/broken_links/detector.py +0 -0
  22. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/config.py +0 -0
  23. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/content/__init__.py +0 -0
  24. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/__init__.py +0 -0
  25. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/detector.py +0 -0
  26. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/extractor.py +0 -0
  27. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/__init__.py +0 -0
  28. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_base.py +0 -0
  29. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_factory.py +0 -0
  30. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_feature_extraction.py +0 -0
  31. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_gliner2.py +0 -0
  32. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_image_classification.py +0 -0
  33. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_llm.py +0 -0
  34. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_object_detection.py +0 -0
  35. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_regex.py +0 -0
  36. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_text_classification.py +0 -0
  37. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/trainer.py +0 -0
  38. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/dependencies.py +0 -0
  39. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/pii/__init__.py +0 -0
  40. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/pii/detector.py +0 -0
  41. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/secrets/__init__.py +0 -0
  42. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/secrets/detector.py +0 -0
  43. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/threat/__init__.py +0 -0
  44. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/threat/code_security_detector.py +0 -0
  45. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/threat/yara_detector.py +0 -0
  46. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/main.py +0 -0
  47. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/models/generated_detectors.py +0 -0
  48. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/models/generated_single_asset_scan_results.py +0 -0
  49. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/__init__.py +0 -0
  50. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/base.py +0 -0
  51. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/console.py +0 -0
  52. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/factory.py +0 -0
  53. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/file.py +0 -0
  54. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/rest.py +0 -0
  55. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/__init__.py +0 -0
  56. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/content_provider.py +0 -0
  57. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/detector_pipeline.py +0 -0
  58. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/parsed_content_provider.py +0 -0
  59. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/worker_pool.py +0 -0
  60. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sandbox/__init__.py +0 -0
  61. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sandbox/runner.py +0 -0
  62. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/__init__.py +0 -0
  63. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/asset_metadata.py +0 -0
  64. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/atlassian_common.py +0 -0
  65. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/azure_blob_storage/__init__.py +0 -0
  66. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/azure_blob_storage/source.py +0 -0
  67. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/base.py +0 -0
  68. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/confluence/__init__.py +0 -0
  69. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/confluence/source.py +0 -0
  70. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/databricks/__init__.py +0 -0
  71. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/databricks/source.py +0 -0
  72. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/delta_lake/__init__.py +0 -0
  73. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/delta_lake/source.py +0 -0
  74. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/dependencies.py +0 -0
  75. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/elasticsearch/__init__.py +0 -0
  76. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/elasticsearch/source.py +0 -0
  77. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/email/__init__.py +0 -0
  78. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/email/source.py +0 -0
  79. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/google_cloud_storage/__init__.py +0 -0
  80. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/google_cloud_storage/source.py +0 -0
  81. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/hive/__init__.py +0 -0
  82. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/hive/source.py +0 -0
  83. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/iceberg/__init__.py +0 -0
  84. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/iceberg/source.py +0 -0
  85. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/jira/__init__.py +0 -0
  86. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/jira/source.py +0 -0
  87. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/kafka/__init__.py +0 -0
  88. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/kafka/source.py +0 -0
  89. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/lakehouse_base.py +0 -0
  90. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/meilisearch/__init__.py +0 -0
  91. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/meilisearch/source.py +0 -0
  92. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mongodb/__init__.py +0 -0
  93. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mongodb/source.py +0 -0
  94. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mssql/__init__.py +0 -0
  95. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mssql/source.py +0 -0
  96. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mysql/__init__.py +0 -0
  97. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mysql/source.py +0 -0
  98. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/neo4j/__init__.py +0 -0
  99. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/neo4j/source.py +0 -0
  100. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/notion/__init__.py +0 -0
  101. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/notion/client.py +0 -0
  102. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/notion/source.py +0 -0
  103. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/object_storage/base.py +0 -0
  104. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/opensearch/__init__.py +0 -0
  105. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/opensearch/source.py +0 -0
  106. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/oracle/__init__.py +0 -0
  107. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/oracle/source.py +0 -0
  108. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/postgresql/__init__.py +0 -0
  109. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/postgresql/source.py +0 -0
  110. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/powerbi/__init__.py +0 -0
  111. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/powerbi/source.py +0 -0
  112. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/recipe_normalizer.py +0 -0
  113. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_client.py +0 -0
  114. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_compatible_storage/README.md +0 -0
  115. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_compatible_storage/__init__.py +0 -0
  116. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_compatible_storage/source.py +0 -0
  117. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/search_engine_base.py +0 -0
  118. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/servicedesk/__init__.py +0 -0
  119. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/servicedesk/source.py +0 -0
  120. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/slack/__init__.py +0 -0
  121. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/slack/source.py +0 -0
  122. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/snowflake/__init__.py +0 -0
  123. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/snowflake/source.py +0 -0
  124. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/sqlite/__init__.py +0 -0
  125. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/sqlite/source.py +0 -0
  126. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tableau/__init__.py +0 -0
  127. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tableau/source.py +0 -0
  128. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tabular_base.py +0 -0
  129. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tabular_utils.py +0 -0
  130. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/wordpress/__init__.py +0 -0
  131. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/wordpress/source.py +0 -0
  132. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/youtube/__init__.py +0 -0
  133. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/youtube/source.py +0 -0
  134. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/telemetry.py +0 -0
  135. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/__init__.py +0 -0
  136. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/content_extraction.py +0 -0
  137. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/dependency_groups.py +0 -0
  138. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/embedded_images.py +0 -0
  139. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/file_metadata.py +0 -0
  140. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/file_parser.py +0 -0
  141. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/file_to_images.py +0 -0
  142. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/hashing.py +0 -0
  143. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/resources.py +0 -0
  144. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/transcription.py +0 -0
  145. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/uv_sync.py +0 -0
  146. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/validation.py +0 -0
  147. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/__init__.py +0 -0
  148. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/_lakehouse_fakes.py +0 -0
  149. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/conftest.py +0 -0
  150. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/__init__.py +0 -0
  151. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
  152. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/conftest.py +0 -0
  153. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/content/__init__.py +0 -0
  154. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/__init__.py +0 -0
  155. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/conftest.py +0 -0
  156. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_invoice_extraction.py +0 -0
  157. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_llm_runner.py +0 -0
  158. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_pipeline_integration.py +0 -0
  159. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_regex_runner.py +0 -0
  160. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_transformer_runners.py +0 -0
  161. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/__init__.py +0 -0
  162. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/conftest.py +0 -0
  163. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/sample_invoice.pdf +0 -0
  164. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/test_pii_detector.py +0 -0
  165. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
  166. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/secrets/__init__.py +0 -0
  167. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/secrets/test_secrets_detector.py +0 -0
  168. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
  169. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_base_detector.py +0 -0
  170. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
  171. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_catalog_commercial.py +0 -0
  172. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_pipeline_types.py +0 -0
  173. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_schema_examples.py +0 -0
  174. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_types.py +0 -0
  175. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_phase2_detectors.py +0 -0
  176. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_registry.py +0 -0
  177. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/threat/__init__.py +0 -0
  178. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/threat/test_code_security_detector.py +0 -0
  179. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/threat/test_yara_detector.py +0 -0
  180. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
  181. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/integration/test_wordpress_links_assets.py +0 -0
  182. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/pipeline/test_detector_pipeline.py +0 -0
  183. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/pipeline/test_worker_pool.py +0 -0
  184. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_assets_metadata_catalog.py +0 -0
  185. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_azure_blob_storage_source.py +0 -0
  186. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_base_source_attachment.py +0 -0
  187. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_base_source_sampling.py +0 -0
  188. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_config.py +0 -0
  189. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_confluence_source.py +0 -0
  190. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_custom_extractor.py +0 -0
  191. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_databricks_source.py +0 -0
  192. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_delta_lake_source.py +0 -0
  193. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_dependency_groups.py +0 -0
  194. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_elasticsearch_source.py +0 -0
  195. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_email_source.py +0 -0
  196. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_google_cloud_storage_source.py +0 -0
  197. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_hashing.py +0 -0
  198. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_hive_source.py +0 -0
  199. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_iceberg_source.py +0 -0
  200. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_jira_source.py +0 -0
  201. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_kafka_source.py +0 -0
  202. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_meilisearch_source.py +0 -0
  203. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_mongodb_source.py +0 -0
  204. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_mssql_source.py +0 -0
  205. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_mysql_source.py +0 -0
  206. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_neo4j_source.py +0 -0
  207. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_notion_source.py +0 -0
  208. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_opensearch_source.py +0 -0
  209. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_oracle_source.py +0 -0
  210. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_outputs.py +0 -0
  211. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_postgresql_source.py +0 -0
  212. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_powerbi_source.py +0 -0
  213. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_recipe_normalizer.py +0 -0
  214. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_s3_compatible_storage_source.py +0 -0
  215. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_sampling_automatic.py +0 -0
  216. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_sandbox_runner.py +0 -0
  217. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_servicedesk_source.py +0 -0
  218. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_slack_source.py +0 -0
  219. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_snowflake_source.py +0 -0
  220. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_source_dependency_groups.py +0 -0
  221. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_sqlite_source.py +0 -0
  222. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_tableau_source.py +0 -0
  223. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_tabular_automatic_sampling.py +0 -0
  224. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_tabular_utils.py +0 -0
  225. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_uv_sync.py +0 -0
  226. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_wordpress_source.py +0 -0
  227. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_youtube_source.py +0 -0
  228. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_youtube_source_integration.py +0 -0
  229. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_content_extraction.py +0 -0
  230. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_embedded_images.py +0 -0
  231. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_file_metadata.py +0 -0
  232. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_file_parser.py +0 -0
  233. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_file_to_images.py +0 -0
  234. {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_transcription.py +0 -0
@@ -0,0 +1,3 @@
1
+ $ uv sync
2
+ Resolved 276 packages in 246ms
3
+ Checked 51 packages in 0.99ms
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: classifyre-cli
3
- Version: 0.4.45
3
+ Version: 0.4.46
4
4
  Summary: Classifyre CLI — scan and classify unstructured data sources
5
5
  License: MIT
6
6
  Keywords: data,ingestion,metadata,pii,secrets,unstructured
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@classifyre/cli",
3
- "version": "0.4.45",
3
+ "version": "0.4.46",
4
4
  "private": true,
5
5
  "scripts": {
6
6
  "build": "uv sync",
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "classifyre-cli"
3
- version = "0.4.45"
3
+ version = "0.4.46"
4
4
  description = "Classifyre CLI — scan and classify unstructured data sources"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12"
@@ -51,6 +51,7 @@ class AssetType(StrEnum):
51
51
  ELASTICSEARCH = 'ELASTICSEARCH'
52
52
  OPENSEARCH = 'OPENSEARCH'
53
53
  MEILISEARCH = 'MEILISEARCH'
54
+ LOCAL_FOLDER = 'LOCAL_FOLDER'
54
55
 
55
56
 
56
57
  class DetectorType(StrEnum):
@@ -355,6 +356,7 @@ class Type(StrEnum):
355
356
  ELASTICSEARCH = 'ELASTICSEARCH'
356
357
  OPENSEARCH = 'OPENSEARCH'
357
358
  MEILISEARCH = 'MEILISEARCH'
359
+ LOCAL_FOLDER = 'LOCAL_FOLDER'
358
360
 
359
361
 
360
362
  class YouTubeRequired(BaseModel):
@@ -644,6 +646,64 @@ class S3CompatibleStorageOptional(BaseModel):
644
646
  scope: ObjectStorageOptionalScope | None = None
645
647
 
646
648
 
649
+ class LocalFolderRequired(BaseModel):
650
+ model_config = ConfigDict(
651
+ extra='forbid',
652
+ )
653
+ path: str = Field(
654
+ ...,
655
+ description='Absolute path of the local folder to scan (for example, /Users/jane/Documents)',
656
+ )
657
+
658
+
659
+ class LocalFolderMasked(BaseModel):
660
+ """
661
+ Local folder scans run with the desktop application's filesystem permissions; no credentials are required.
662
+ """
663
+
664
+ model_config = ConfigDict(
665
+ extra='forbid',
666
+ )
667
+
668
+
669
+ class LocalFolderOptionalTraversal(BaseModel):
670
+ """
671
+ Directory traversal controls.
672
+ """
673
+
674
+ model_config = ConfigDict(
675
+ extra='forbid',
676
+ )
677
+ follow_symlinks: bool | None = Field(
678
+ False,
679
+ description='Follow symbolic links while traversing (loops are not detected; leave off unless needed)',
680
+ )
681
+ include_hidden: bool | None = Field(
682
+ False,
683
+ description='Include hidden files and dot-directories (names starting with a dot)',
684
+ )
685
+ max_depth: int | None = Field(
686
+ None,
687
+ description='Maximum directory depth below the selected folder (unset scans all levels)',
688
+ ge=1,
689
+ le=64,
690
+ )
691
+ max_file_bytes: int | None = Field(
692
+ 10485760,
693
+ description='Maximum bytes read per file for MIME detection and text extraction',
694
+ ge=1024,
695
+ le=104857600,
696
+ )
697
+
698
+
699
+ class LocalFolderOptional(BaseModel):
700
+ model_config = ConfigDict(
701
+ extra='forbid',
702
+ )
703
+ traversal: LocalFolderOptionalTraversal | None = None
704
+ scope: ObjectStorageOptionalScope | None = None
705
+
706
+
647
707
  class AzureBlobStorageRequired(BaseModel):
648
708
  model_config = ConfigDict(
649
709
  extra='forbid',
@@ -2100,6 +2160,24 @@ class S3CompatibleStorageInput(CoreInput):
2100
2160
  resources: ResourceOverrides | None = None
2101
2161
 
2102
2162
 
2163
+ class LocalFolderInput(CoreInput):
2164
+ type: Literal['LOCAL_FOLDER'] | None = Field(
2165
+ None, description='Type of the asset or source'
2166
+ )
2167
+ required: LocalFolderRequired
2168
+ masked: LocalFolderMasked | None = None
2169
+ optional: LocalFolderOptional | None = None
2170
+ detectors: list[Detector] | None = Field(
2171
+ None, description='Detectors to run on ingested content'
2172
+ )
2173
+ custom_detectors: list[CustomDetectorSelection] | None = Field(
2174
+ None,
2175
+ description='Reusable custom detector IDs selected from the custom detector catalog.',
2176
+ )
2177
+ sampling: SamplingConfig
2178
+ resources: ResourceOverrides | None = None
2179
+
2180
+
2103
2181
  class AzureBlobStorageInput(CoreInput):
2104
2182
  type: Literal['AZURE_BLOB_STORAGE'] | None = Field(
2105
2183
  None, description='Type of the asset or source'
@@ -2531,7 +2609,7 @@ class ConfluenceOptionalConnection(BaseModel):
2531
2609
  )
2532
2610
 
2533
2611
 
2534
- class Type18(StrEnum):
2612
+ class Type19(StrEnum):
2535
2613
  """
2536
2614
  Filter spaces by space type
2537
2615
  """
@@ -2568,7 +2646,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
2568
2646
  keys: list[str] | None = Field(
2569
2647
  None, description='Filter spaces by keys (up to 250)', max_length=250
2570
2648
  )
2571
- type: Type18 | None = Field(None, description='Filter spaces by space type')
2649
+ type: Type19 | None = Field(None, description='Filter spaces by space type')
2572
2650
  status: Status | None = Field(None, description='Filter spaces by status')
2573
2651
  labels: list[str] | None = Field(
2574
2652
  None,
@@ -2834,7 +2912,7 @@ class ServiceDeskOptional(BaseModel):
2834
2912
  content: ServiceDeskOptionalContent | None = None
2835
2913
 
2836
2914
 
2837
- class Type19(StrEnum):
2915
+ class Type20(StrEnum):
2838
2916
  """
2839
2917
  Type of the asset or source
2840
2918
  """
@@ -2868,6 +2946,7 @@ class Type19(StrEnum):
2868
2946
  ELASTICSEARCH = 'ELASTICSEARCH'
2869
2947
  OPENSEARCH = 'OPENSEARCH'
2870
2948
  MEILISEARCH = 'MEILISEARCH'
2949
+ LOCAL_FOLDER = 'LOCAL_FOLDER'
2871
2950
 
2872
2951
 
2873
2952
  class ConfluenceInput(CoreInput):
@@ -3682,6 +3761,7 @@ class SourceInput(
3682
3761
  RootModel[
3683
3762
  SlackInput
3684
3763
  | S3CompatibleStorageInput
3764
+ | LocalFolderInput
3685
3765
  | AzureBlobStorageInput
3686
3766
  | GoogleCloudStorageInput
3687
3767
  | PostgreSQLInput
@@ -3714,6 +3794,7 @@ class SourceInput(
3714
3794
  root: (
3715
3795
  SlackInput
3716
3796
  | S3CompatibleStorageInput
3797
+ | LocalFolderInput
3717
3798
  | AzureBlobStorageInput
3718
3799
  | GoogleCloudStorageInput
3719
3800
  | PostgreSQLInput
@@ -0,0 +1,3 @@
1
+ from .source import LocalFolderSource
2
+
3
+ __all__ = ["LocalFolderSource"]
@@ -0,0 +1,146 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import os
5
+ from collections.abc import Iterator
6
+ from datetime import UTC, datetime
7
+ from pathlib import Path
8
+ from typing import Any
9
+ from urllib.parse import quote
10
+
11
+ from ...models.generated_input import LocalFolderInput
12
+ from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ class LocalFolderSource(ObjectStorageSourceBase):
18
+ """Scans a folder on the local filesystem (desktop application deployments).
19
+
20
+ Reuses the object-storage pipeline: each file becomes an ObjectRef whose key
21
+ is the path relative to the scanned root, so sampling, MIME resolution, text
22
+ extraction, file metadata, and embedded-image child assets all come from
23
+ ObjectStorageSourceBase.
24
+ """
25
+
26
+ source_type = "local_folder"
27
+ provider_label = "LOCAL_FOLDER"
28
+ input_model = LocalFolderInput
29
+
30
+ DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024
31
+
32
+ _resolved_root: Path | None = None
33
+
34
+ def _root(self) -> Path:
35
+ if self._resolved_root is not None:
36
+ return self._resolved_root
37
+ raw = str(self.config.required.path).strip()
38
+ if not raw:
39
+ raise ValueError("required.path must be set")
40
+ root = Path(raw).expanduser()
41
+ if not root.is_absolute():
42
+ raise ValueError(f"required.path must be an absolute path, got: {raw}")
43
+ if not root.exists():
44
+ raise ValueError(f"Folder does not exist: {root}")
45
+ if not root.is_dir():
46
+ raise ValueError(f"Path is not a folder: {root}")
47
+ self._resolved_root = root.resolve()
48
+ return self._resolved_root
49
+
50
+ def _traversal_option(self, key: str, default: Any = None) -> Any:
51
+ optional = self.config.optional
52
+ traversal = getattr(optional, "traversal", None) if optional else None
53
+ if traversal is not None:
54
+ value = getattr(traversal, key, None)
55
+ if value is not None:
56
+ return value
57
+ return default
58
+
59
+ def _follow_symlinks(self) -> bool:
60
+ return bool(self._traversal_option("follow_symlinks", False))
61
+
62
+ def _include_hidden(self) -> bool:
63
+ return bool(self._traversal_option("include_hidden", False))
64
+
65
+ def _max_depth(self) -> int | None:
66
+ value = self._traversal_option("max_depth")
67
+ try:
68
+ return int(value) if value is not None else None
69
+ except (TypeError, ValueError):
70
+ return None
71
+
72
+ def _max_file_bytes(self) -> int:
73
+ value = self._traversal_option("max_file_bytes", self.DEFAULT_MAX_FILE_BYTES)
74
+ try:
75
+ parsed = int(value)
76
+ except (TypeError, ValueError):
77
+ return self.DEFAULT_MAX_FILE_BYTES
78
+ return max(parsed, 1024)
79
+
80
+ def _is_hidden(self, name: str) -> bool:
81
+ return name.startswith(".")
82
+
83
+ def _walk(self, directory: Path, depth: int) -> Iterator[ObjectRef]:
84
+ max_depth = self._max_depth()
85
+ follow_symlinks = self._follow_symlinks()
86
+ include_hidden = self._include_hidden()
87
+
88
+ try:
89
+ entries = sorted(os.scandir(directory), key=lambda entry: entry.name)
90
+ except OSError as exc:
91
+ logger.warning("Skipping unreadable directory %s: %s", directory, exc)
92
+ return
93
+
94
+ for entry in entries:
95
+ if not include_hidden and self._is_hidden(entry.name):
96
+ continue
97
+ try:
98
+ if entry.is_dir(follow_symlinks=follow_symlinks):
99
+ if max_depth is None or depth < max_depth:
100
+ yield from self._walk(Path(entry.path), depth + 1)
101
+ continue
102
+ if not entry.is_file(follow_symlinks=follow_symlinks):
103
+ continue
104
+ stat = entry.stat(follow_symlinks=follow_symlinks)
105
+ except OSError as exc:
106
+ logger.warning("Skipping unreadable entry %s: %s", entry.path, exc)
107
+ continue
108
+
109
+ key = Path(entry.path).relative_to(self._root()).as_posix()
110
+ if not self._object_matches_extension_filters(key):
111
+ continue
112
+ prefix = self._prefix()
113
+ if prefix and not key.startswith(prefix):
114
+ continue
115
+ if stat.st_size == 0 and not self._include_empty_objects():
116
+ continue
117
+
118
+ yield ObjectRef(
119
+ key=key,
120
+ size=stat.st_size,
121
+ last_modified=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
122
+ # Change signature: mtime+size, so edits produce a new checksum
123
+ # without hashing file bytes during discovery.
124
+ etag=f"{stat.st_mtime_ns:x}-{stat.st_size:x}",
125
+ )
126
+
127
+ def _list_objects(self) -> Iterator[ObjectRef]:
128
+ yield from self._walk(self._root(), depth=0)
129
+
130
+ def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
131
+ file_path = self._root() / ref.key
132
+ max_bytes = self._max_file_bytes()
133
+ with open(file_path, "rb") as handle:
134
+ file_bytes = handle.read(max_bytes)
135
+ if ref.size > max_bytes:
136
+ logger.info(
137
+ "Truncated %s to %d of %d bytes for content extraction",
138
+ file_path,
139
+ max_bytes,
140
+ ref.size,
141
+ )
142
+ return file_bytes, None
143
+
144
+ def _external_url(self, key: str) -> str:
145
+ absolute = (self._root() / key).as_posix()
146
+ return f"file://{quote(absolute)}"
@@ -0,0 +1,335 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from pathlib import Path
5
+
6
+ import pytest
7
+
8
+ from src.models.generated_single_asset_scan_results import AssetType as OutputAssetType
9
+ from src.sources.local_folder.source import LocalFolderSource
10
+
11
+ # Minimal valid 1x1 PNG (8-byte signature + IHDR/IDAT/IEND chunks).
12
+ _PNG_BYTES = bytes.fromhex(
13
+ "89504e470d0a1a0a0000000d494844520000000100000001080600000"
14
+ "01f15c4890000000a49444154789c6300010000050001a5f645400000000049454e44ae426082"
15
+ )
16
+
17
+
18
+ def _recipe(path: Path, *, optional: dict | None = None, sampling: dict | None = None) -> dict:
19
+ return {
20
+ "type": "LOCAL_FOLDER",
21
+ "required": {"path": str(path)},
22
+ "masked": {},
23
+ "optional": optional or {},
24
+ "sampling": sampling or {"strategy": "ALL"},
25
+ }
26
+
27
+
28
+ def _make_source(path: Path, *, optional: dict | None = None, sampling: dict | None = None):
29
+ return LocalFolderSource(
30
+ _recipe(path, optional=optional, sampling=sampling),
31
+ source_id="s",
32
+ runner_id="r",
33
+ )
34
+
35
+
36
+ async def _extract_assets(source: LocalFolderSource) -> list:
37
+ assets = []
38
+ async for batch in source.extract_raw():
39
+ assets.extend(batch)
40
+ return assets
41
+
42
+
43
+ def _keys(assets: list) -> set[str]:
44
+ return {a.metadata["object_key"] for a in assets}
45
+
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # 1. Nested traversal
49
+ # ---------------------------------------------------------------------------
50
+
51
+
52
+ @pytest.mark.asyncio
53
+ async def test_nested_traversal_returns_posix_relative_keys(tmp_path: Path):
54
+ (tmp_path / "top.txt").write_text("top level")
55
+ nested_dir = tmp_path / "a" / "b"
56
+ nested_dir.mkdir(parents=True)
57
+ (nested_dir / "deep.txt").write_text("deep file")
58
+
59
+ source = _make_source(tmp_path)
60
+ assets = await _extract_assets(source)
61
+
62
+ assert _keys(assets) == {"top.txt", "a/b/deep.txt"}
63
+ names = {a.name for a in assets}
64
+ assert names == {"top.txt", "deep.txt"}
65
+
66
+
67
+ # ---------------------------------------------------------------------------
68
+ # 2. Hidden files/dirs
69
+ # ---------------------------------------------------------------------------
70
+
71
+
72
+ @pytest.mark.asyncio
73
+ async def test_hidden_files_skipped_by_default(tmp_path: Path):
74
+ (tmp_path / "visible.txt").write_text("visible")
75
+ (tmp_path / ".hidden.txt").write_text("hidden")
76
+ hidden_dir = tmp_path / ".hiddendir"
77
+ hidden_dir.mkdir()
78
+ (hidden_dir / "inside.txt").write_text("inside hidden dir")
79
+
80
+ source = _make_source(tmp_path)
81
+ assets = await _extract_assets(source)
82
+
83
+ assert _keys(assets) == {"visible.txt"}
84
+
85
+
86
+ @pytest.mark.asyncio
87
+ async def test_hidden_files_included_when_configured(tmp_path: Path):
88
+ (tmp_path / "visible.txt").write_text("visible")
89
+ (tmp_path / ".hidden.txt").write_text("hidden")
90
+ hidden_dir = tmp_path / ".hiddendir"
91
+ hidden_dir.mkdir()
92
+ (hidden_dir / "inside.txt").write_text("inside hidden dir")
93
+
94
+ source = _make_source(tmp_path, optional={"traversal": {"include_hidden": True}})
95
+ assets = await _extract_assets(source)
96
+
97
+ assert _keys(assets) == {"visible.txt", ".hidden.txt", ".hiddendir/inside.txt"}
98
+
99
+
100
+ # ---------------------------------------------------------------------------
101
+ # 3. include/exclude extensions
102
+ # ---------------------------------------------------------------------------
103
+
104
+
105
+ @pytest.mark.asyncio
106
+ async def test_include_extensions_filters_to_allowlist(tmp_path: Path):
107
+ (tmp_path / "a.txt").write_text("a")
108
+ (tmp_path / "b.csv").write_text("b,c\n1,2\n")
109
+ (tmp_path / "c.md").write_text("# markdown")
110
+
111
+ source = _make_source(tmp_path, optional={"scope": {"include_extensions": [".txt", ".csv"]}})
112
+ assets = await _extract_assets(source)
113
+
114
+ assert _keys(assets) == {"a.txt", "b.csv"}
115
+
116
+
117
+ @pytest.mark.asyncio
118
+ async def test_exclude_extensions_filters_out_denylist(tmp_path: Path):
119
+ (tmp_path / "a.txt").write_text("a")
120
+ (tmp_path / "b.csv").write_text("b,c\n1,2\n")
121
+ (tmp_path / "c.md").write_text("# markdown")
122
+
123
+ source = _make_source(tmp_path, optional={"scope": {"exclude_extensions": [".md"]}})
124
+ assets = await _extract_assets(source)
125
+
126
+ assert _keys(assets) == {"a.txt", "b.csv"}
127
+
128
+
129
+ # ---------------------------------------------------------------------------
130
+ # 4. Empty files
131
+ # ---------------------------------------------------------------------------
132
+
133
+
134
+ @pytest.mark.asyncio
135
+ async def test_empty_files_skipped_by_default(tmp_path: Path):
136
+ (tmp_path / "empty.txt").touch()
137
+ (tmp_path / "nonempty.txt").write_text("content")
138
+
139
+ source = _make_source(tmp_path)
140
+ assets = await _extract_assets(source)
141
+
142
+ assert _keys(assets) == {"nonempty.txt"}
143
+
144
+
145
+ @pytest.mark.asyncio
146
+ async def test_empty_files_included_when_configured(tmp_path: Path):
147
+ (tmp_path / "empty.txt").touch()
148
+ (tmp_path / "nonempty.txt").write_text("content")
149
+
150
+ source = _make_source(tmp_path, optional={"scope": {"include_empty_objects": True}})
151
+ assets = await _extract_assets(source)
152
+
153
+ assert _keys(assets) == {"empty.txt", "nonempty.txt"}
154
+
155
+
156
+ # ---------------------------------------------------------------------------
157
+ # 5. max_depth
158
+ # ---------------------------------------------------------------------------
159
+
160
+
161
+ @pytest.mark.asyncio
162
+ async def test_max_depth_limits_recursion(tmp_path: Path):
163
+ (tmp_path / "root.txt").write_text("root")
164
+ level1 = tmp_path / "l1"
165
+ level1.mkdir()
166
+ (level1 / "one.txt").write_text("one")
167
+ level2 = level1 / "l2"
168
+ level2.mkdir()
169
+ (level2 / "two.txt").write_text("two")
170
+
171
+ source = _make_source(tmp_path, optional={"traversal": {"max_depth": 1}})
172
+ assets = await _extract_assets(source)
173
+
174
+ assert _keys(assets) == {"root.txt", "l1/one.txt"}
175
+
176
+
177
+ # ---------------------------------------------------------------------------
178
+ # 6. max_file_bytes truncation
179
+ # ---------------------------------------------------------------------------
180
+
181
+
182
+ def test_download_object_truncates_to_max_file_bytes(tmp_path: Path):
183
+ big_file = tmp_path / "big.bin"
184
+ big_file.write_bytes(b"x" * 3000)
185
+
186
+ source = _make_source(tmp_path, optional={"traversal": {"max_file_bytes": 1024}})
187
+ refs = list(source._list_objects())
188
+ assert len(refs) == 1
189
+
190
+ file_bytes, content_type_hint = source._download_object(refs[0])
191
+
192
+ assert len(file_bytes) == 1024
193
+ assert content_type_hint is None
194
+
195
+
196
+ # ---------------------------------------------------------------------------
197
+ # 7. external_url / hash determinism / etag & checksum change on edit
198
+ # ---------------------------------------------------------------------------
199
+
200
+
201
+ @pytest.mark.asyncio
202
+ async def test_external_url_and_hash_determinism(tmp_path: Path):
203
+ file_path = tmp_path / "note.txt"
204
+ file_path.write_text("hello")
205
+
206
+ source_a = _make_source(tmp_path)
207
+ assets_a = await _extract_assets(source_a)
208
+ assert len(assets_a) == 1
209
+ asset_a = assets_a[0]
210
+
211
+ assert asset_a.external_url.startswith("file://")
212
+ assert asset_a.external_url == f"file://{file_path.resolve().as_posix()}"
213
+
214
+ source_b = _make_source(tmp_path)
215
+ assets_b = await _extract_assets(source_b)
216
+ asset_b = assets_b[0]
217
+
218
+ assert asset_a.hash == asset_b.hash
219
+
220
+
221
+ @pytest.mark.asyncio
222
+ async def test_etag_and_checksum_change_on_content_and_mtime_change(tmp_path: Path):
223
+ file_path = tmp_path / "note.txt"
224
+ file_path.write_text("hello")
225
+ os.utime(file_path, (1_700_000_000, 1_700_000_000))
226
+
227
+ source_before = _make_source(tmp_path)
228
+ assets_before = await _extract_assets(source_before)
229
+ asset_before = assets_before[0]
230
+
231
+ file_path.write_text("hello world, now longer")
232
+ os.utime(file_path, (1_700_001_000, 1_700_001_000))
233
+
234
+ source_after = _make_source(tmp_path)
235
+ assets_after = await _extract_assets(source_after)
236
+ asset_after = assets_after[0]
237
+
238
+ # Hash is derived from the (unchanged) external_url/key, so it stays stable...
239
+ assert asset_before.hash == asset_after.hash
240
+ # ...but the etag (mtime+size signature) and checksum must reflect the edit.
241
+ assert asset_before.metadata["etag"] != asset_after.metadata["etag"]
242
+ assert asset_before.checksum != asset_after.checksum
243
+
244
+
245
+ # ---------------------------------------------------------------------------
246
+ # 8. asset_type mapping
247
+ # ---------------------------------------------------------------------------
248
+
249
+
250
+ @pytest.mark.asyncio
251
+ async def test_asset_type_mapping_by_extension(tmp_path: Path):
252
+ (tmp_path / "doc.txt").write_text("plain text")
253
+ (tmp_path / "table.csv").write_text("a,b\n1,2\n")
254
+ (tmp_path / "image.png").write_bytes(_PNG_BYTES)
255
+
256
+ source = _make_source(tmp_path)
257
+ assets = await _extract_assets(source)
258
+ by_key = {a.metadata["object_key"]: a for a in assets}
259
+
260
+ assert by_key["doc.txt"].asset_type == OutputAssetType.TXT
261
+ assert by_key["table.csv"].asset_type == OutputAssetType.TABLE
262
+ assert by_key["image.png"].asset_type == OutputAssetType.IMAGE
263
+ assert by_key["image.png"].asset_kind == "image"
264
+
265
+
266
+ # ---------------------------------------------------------------------------
267
+ # 9. Metadata contract (asset_kind + catalog-validated metadata)
268
+ # ---------------------------------------------------------------------------
269
+
270
+
271
+ @pytest.mark.asyncio
272
+ async def test_metadata_contract_for_file_asset(tmp_path: Path):
273
+ (tmp_path / "doc.txt").write_text("plain text")
274
+
275
+ source = _make_source(tmp_path)
276
+ assets = await _extract_assets(source)
277
+ asset = assets[0]
278
+
279
+ assert asset.asset_kind == "file"
280
+ assert asset.metadata["provider"] == "LOCAL_FOLDER"
281
+ assert asset.metadata["object_key"] == "doc.txt"
282
+ assert asset.metadata["size_bytes"] == len("plain text")
283
+ assert "mime_type" in asset.metadata
284
+
285
+
286
+ # ---------------------------------------------------------------------------
287
+ # 10. test_connection
288
+ # ---------------------------------------------------------------------------
289
+
290
+
291
+ def test_connection_success_for_valid_directory(tmp_path: Path):
292
+ (tmp_path / "a.txt").write_text("a")
293
+
294
+ source = _make_source(tmp_path)
295
+ result = source.test_connection()
296
+
297
+ assert result["status"] == "SUCCESS"
298
+
299
+
300
+ def test_connection_failure_for_missing_directory(tmp_path: Path):
301
+ missing = tmp_path / "does-not-exist"
302
+
303
+ source = _make_source(missing)
304
+ result = source.test_connection()
305
+
306
+ assert result["status"] == "FAILURE"
307
+
308
+
309
+ # ---------------------------------------------------------------------------
310
+ # 11. Sampling
311
+ # ---------------------------------------------------------------------------
312
+
313
+
314
+ @pytest.mark.asyncio
315
+ async def test_sampling_random_respects_rows_per_page(tmp_path: Path):
316
+ # SamplingConfig.rows_per_page has a hard floor of 10 (ge=10), so exercise the
317
+ # floor value against a larger population rather than requesting 1 row.
318
+ for i in range(15):
319
+ (tmp_path / f"file{i}.txt").write_text(f"content {i}")
320
+
321
+ source = _make_source(tmp_path, sampling={"strategy": "RANDOM", "rows_per_page": 10})
322
+ assets = await _extract_assets(source)
323
+
324
+ assert len(assets) == 10
325
+
326
+
327
+ @pytest.mark.asyncio
328
+ async def test_sampling_latest_respects_rows_per_page(tmp_path: Path):
329
+ for i in range(15):
330
+ (tmp_path / f"file{i}.txt").write_text(f"content {i}")
331
+
332
+ source = _make_source(tmp_path, sampling={"strategy": "LATEST", "rows_per_page": 10})
333
+ assets = await _extract_assets(source)
334
+
335
+ assert len(assets) == 10
@@ -721,7 +721,7 @@ wheels = [
721
721
 
722
722
  [[package]]
723
723
  name = "classifyre-cli"
724
- version = "0.4.45"
724
+ version = "0.4.46"
725
725
  source = { editable = "." }
726
726
  dependencies = [
727
727
  { name = "beautifulsoup4" },
@@ -1151,7 +1151,7 @@ youtube = [
1151
1151
 
1152
1152
  [[package]]
1153
1153
  name = "classifyre-schemas"
1154
- version = "0.4.45"
1154
+ version = "0.4.46"
1155
1155
  source = { editable = "../../packages/schemas" }
1156
1156
  dependencies = [
1157
1157
  { name = "fastjsonschema" },
@@ -1,3 +0,0 @@
1
- $ uv sync
2
- Resolved 276 packages in 177ms
3
- Checked 51 packages in 1ms