topos-node 1.0.1__tar.gz → 1.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {topos_node-1.0.1 → topos_node-1.0.5}/PKG-INFO +11 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/README.md +10 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/pyproject.toml +1 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/__version__.py +2 -2
- topos_node-1.0.5/topos/api/engine_run.py +43 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/enrichment.py +3 -3
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/app.py +2 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/config/settings.py +18 -3
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/handlers.py +18 -18
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/logging.py +56 -2
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/disclosure/privacy_layer.py +10 -10
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/__init__.py +9 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/huggingface.py +139 -77
- topos_node-1.0.5/topos/engine/client.py +85 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/engine.py +28 -3
- topos_node-1.0.5/topos/engine/memory_utils.py +57 -0
- topos_node-1.0.5/topos/engine/model_cache.py +211 -0
- topos_node-1.0.5/topos/engine/pipeline_memory.py +26 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/queue_manager.py +22 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/registration.py +15 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/_engine_runner.py +6 -3
- topos_node-1.0.5/topos/enrichment/jobs/canonical/emo_27_job.py +111 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/topic_clusters_job.py +2 -2
- topos_node-1.0.5/topos/enrichment/models/manager.py +37 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/orchestrator.py +33 -10
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/canonical_pipeline.py +7 -2
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/ingest_helpers.py +7 -7
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/manager.py +5 -5
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/observability/metrics.py +9 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/pipeline/envelope.py +1 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/pipeline/stub_enqueue.py +1 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/inference.py +4 -4
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/manifest_validation.py +51 -6
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sanitization/nsfw_classifier.py +7 -12
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sanitization/privacy_filter.py +10 -13
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/scrub_service.py +1 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/raw/browser_flat_tables.py +3 -3
- {topos_node-1.0.1 → topos_node-1.0.5}/topos_node.egg-info/PKG-INFO +11 -1
- {topos_node-1.0.1 → topos_node-1.0.5}/topos_node.egg-info/SOURCES.txt +5 -0
- topos_node-1.0.1/topos/enrichment/jobs/canonical/emo_27_job.py +0 -99
- topos_node-1.0.1/topos/enrichment/models/manager.py +0 -8
- {topos_node-1.0.1 → topos_node-1.0.5}/LICENSE +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/setup.cfg +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/shared/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/shared/filtering.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/shared/schema_registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/tests/test_llm_integrations_storage.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/duckdb_adapter.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/messenger_communities.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/messenger_graph.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/messenger_labels.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/profiles.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/query_engine.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/analytics/raw_queries.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/analytics.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/app_registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/backup.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/compute_remote.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/data_commit.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/data_explorer_table_prefs.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/database_explorer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/db.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/device.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/filter_lab.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/health.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/home_chat.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/ingestion_api.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/ingestion_compat.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/ingestion_sources.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/llm.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/local_mcp.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/messenger_analytics.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/privacy_disclose.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/query_api.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/runtime_bootstrap.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/sanitization_ollama_config.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/signal.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/signal_extraction_config.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/source_install.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/source_scrub.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/sources.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/sources_overview.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/sync.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/ui_config.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/uma_data.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/usage.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/api/user_identity.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/auth.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/browser_activity_mapper.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/chatgpt_mapper.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/demo_mappers.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/grok_mapper.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/mappers/messenger_mapper.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/models.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/canonicalization/resolver.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/cli/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/cli/__main__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/cli/commands.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/config/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/config/sanitization_ollama.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/config/signal_extraction.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/contacts/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/contacts/identity.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/control_plane_client.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/api_models.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/connection_resilience.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/device_helpers.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/errors.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/events.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/metrics.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/routine_access.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/startup_banner.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/state.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/table_layers.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/core/types.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/data_explorer_row_delete.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/data_explorer_table_prefs.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/data_explorer_tables.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/disclosure/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/disclosure/canonical_writer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/disclosure/content_policy.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/disclosure/field_registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/disclosure/tier.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/generative_prompts.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/generative_response.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/ollama.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/openai.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/openai_compatible.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/redpill.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/backends/stub.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/intake.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/result_formatter.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/router.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/scoped_token.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/tasks.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/transport.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/usage_guard.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/usage_observation.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/engine/validator.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/derived_tables.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/job_writer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/_batch_limits.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/availability_scores_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/brief_fallback.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/dimension_summary_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/embeddings_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/entities_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/goal_extraction_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/pii_redaction_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/relationship_edges_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/sentiment_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/topics_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/url_classification_core.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/url_classification_enrichment_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/canonical/url_classification_signal_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/raw/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/raw/attachments_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/raw/language_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/raw/time_normalization_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/jobs/raw/tool_calls_job.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/models/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/models/mvp_defaults.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/models/registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/models/versioning.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/processor.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/progress_bar.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/registry_contract.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/enrichment/website_classifier.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/fit/evaluator.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/brief_canonical_loader.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/brief_ontology.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/brief_renderer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/brief_schemas.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/chunking.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/data_health.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/dimension_briefs.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/dimension_definition_loader.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/dimension_profiles.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/dimension_registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/extraction/artifact_router.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/extraction/artifact_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/extraction/rule_extractors.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/graph_sanitize.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/hybrid_search.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/query_embed_cache.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/schemas.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/service.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/signal_object_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/source_hydration.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/structured_signal.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/topic_clustering.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/typed_stores/aggregates.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/typed_stores/scope_materializer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/vector_codec.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/vector_math.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/features/signal/vector_settings.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/filter_lab/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/filter_lab/bundles.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/filter_lab/schema.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/filter_lab/service.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/filter_lab/store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/filter_lab/worker.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/home_chat/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/home_chat/schema.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/home_chat/store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/hosted_pool_lease.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/checkpoints/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/checkpoints/checkpoint_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/checkpoints/sqlite_checkpoint_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/inbox_crypto.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/inbox_drain.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/jobs.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/journal_location_fanout.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/journal_time_log_normalize.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/local_sync.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/log_preview.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/browser_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/calendar_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/chatgpt_conversation_flattener.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/chatgpt_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/demo_file_parsers.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/grok_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/messenger_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/parsers/voxterm_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/progress.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/reprocess.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/calendar.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/chatgpt.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/contact_importers.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/grok.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/imessage_reader.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/signal_export_parser.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/sources/signal_reader.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/state_machine.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/triggers/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/triggers/file_trigger.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/triggers/sqlite_trigger.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/usage_inbox_dedupe.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/validation/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/validation/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/validation/schema_registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/ingestion/validation/schema_validator.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/lineage/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/lineage/provenance.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/lineage/tracker.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/llm_integrations_storage.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/mcp_stdio_proxy.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/observability/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/observability/alerts.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/observability/tracing.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/openai_client.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/pipeline/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/pipeline/audit.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/pipeline/stages.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/projections/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/projections/vector_index/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/projections/vector_index/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/projections/vector_index/builders.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/projections/vector_index/health_checks.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/audit.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/disclosure.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/fingerprint.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/game_layer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/intent.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/manifest.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/pipeline.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/retrieval.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/runtime.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/scope_registry_loader.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/session.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/session_utils.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/source_generation.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/turn_classifier.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/query/types.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/rate_limit.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/routines/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/routines/schema.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/routines/store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/runtime_update.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sanitization/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sanitization/ollama_transforms.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/scope_resolution.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/container.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/embeddings/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/embeddings/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/embeddings/local.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/embeddings/remote.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/interfaces.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/llm/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/llm/base.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/llm/openai.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/local.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/services/postgres.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/bundled_canonical_triples.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/canonical_signal_defaults.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/definitions.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/install_service.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/registry.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/runtime_install.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sources/scrub_attribution.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/startup_banner.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/factory.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/fakes.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/postgres/stores.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/protocols.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/sqlite/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/sqlite/stores.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/adapters/sqlite/vector_search.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/activity_tables.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/ai_chat/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/ai_chat/canonicalizer.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/ai_chat/mapper.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/ai_chat/model.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/ai_chat/tables.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/calendar_contacts_tables.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/canonical_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/conversations_tables.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/mapping_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/canonical/postgres.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/client.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/canonical_disclosure_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/canonical_nsfw_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/extraction_artifacts.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/journal_entries_duration_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/journal_entries_ends_at_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/journal_entries_people_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/journal_entries_place_name_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/journal_entries_starts_at_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/remediation_person_model.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/scope_source_generation.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/signal_dimension_harness.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/signal_objects.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/signal_objects_updated_by_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/stage9_column_renames.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/vector_storage_v1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/vector_storage_v2.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/vector_storage_v3.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_phase0.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_phase1.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_phase4_messages_cutover.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_phase5_topic_clusters.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_phase6_dimension_briefs.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_query_quality.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/migrations/wiki_mvp_topic_clusters_coordination.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/paths.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/postgres.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/schema.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/db/storage_breakdown.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/enrichment/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/enrichment/canonical_enrichment_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/enrichment/raw_enrichment_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/normalized/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/normalized/normalized_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/oplog/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/oplog/decision.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/oplog/oplog_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/oplog/postgres.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/person/identity_resolver.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/projections/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/projections/index_ops_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/projections/vector_index_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/raw/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/raw/file_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/raw/raw_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/raw/raw_tables_manager.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/raw/sqlite_raw_store.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/security/encryption.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/signal_identity.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/source_settings.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/storage/user_identity.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sync/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sync/client.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/sync_handlers.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/testing/__init__.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/testing/lifespan.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/uma_contact_enrichment.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/uma_filters.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/uma_resource_id.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/uma_rpt.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/utils/base_object.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos/websocket_client.py +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos_node.egg-info/dependency_links.txt +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos_node.egg-info/entry_points.txt +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos_node.egg-info/requires.txt +0 -0
- {topos_node-1.0.1 → topos_node-1.0.5}/topos_node.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: topos-node
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.5
|
|
4
4
|
Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
|
|
5
5
|
License-Expression: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://topos.dialogues.ai
|
|
@@ -195,6 +195,16 @@ uv sync --extra engine
|
|
|
195
195
|
just run
|
|
196
196
|
```
|
|
197
197
|
|
|
198
|
+
### Engine memory (local dev)
|
|
199
|
+
|
|
200
|
+
ML models are cached inside the Engine with LRU eviction. For lighter local runs (especially inside Cursor's integrated terminal), see [Engine memory management](topos/docs/ml-manager-v2/MEMORY_MANAGEMENT.md).
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
# Default: ENGINE_MAX_RESIDENT_MODELS=3; pipeline flush is automatic
|
|
204
|
+
# Override only if needed, e.g. lower RAM: export ENGINE_MAX_RESIDENT_MODELS=2
|
|
205
|
+
export PRIVACY_FILTER_DEVICE=cpu
|
|
206
|
+
```
|
|
207
|
+
|
|
198
208
|
Run tests:
|
|
199
209
|
|
|
200
210
|
```bash
|
|
@@ -139,6 +139,16 @@ uv sync --extra engine
|
|
|
139
139
|
just run
|
|
140
140
|
```
|
|
141
141
|
|
|
142
|
+
### Engine memory (local dev)
|
|
143
|
+
|
|
144
|
+
ML models are cached inside the Engine with LRU eviction. For lighter local runs (especially inside Cursor's integrated terminal), see [Engine memory management](topos/docs/ml-manager-v2/MEMORY_MANAGEMENT.md).
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
# Default: ENGINE_MAX_RESIDENT_MODELS=3; pipeline flush is automatic
|
|
148
|
+
# Override only if needed, e.g. lower RAM: export ENGINE_MAX_RESIDENT_MODELS=2
|
|
149
|
+
export PRIVACY_FILTER_DEVICE=cpu
|
|
150
|
+
```
|
|
151
|
+
|
|
142
152
|
Run tests:
|
|
143
153
|
|
|
144
154
|
```bash
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "topos-node"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.5"
|
|
8
8
|
description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Engine task execution HTTP endpoint."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict
|
|
6
|
+
|
|
7
|
+
from fastapi import APIRouter, Depends, HTTPException, status
|
|
8
|
+
|
|
9
|
+
from ..auth import require_api_key
|
|
10
|
+
from ..engine.engine import Engine
|
|
11
|
+
from ..engine.tasks import ProcessingResult, ProcessingTask
|
|
12
|
+
|
|
13
|
+
router = APIRouter(tags=["engine-run"])
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@router.post("/v1/engine/run", dependencies=[Depends(require_api_key)])
|
|
17
|
+
async def engine_run(body: Dict[str, Any]) -> Dict[str, Any]:
|
|
18
|
+
"""Execute a ProcessingTask synchronously and return ProcessingResult JSON."""
|
|
19
|
+
try:
|
|
20
|
+
task = ProcessingTask.model_validate(body)
|
|
21
|
+
except Exception as exc:
|
|
22
|
+
raise HTTPException(
|
|
23
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
24
|
+
detail=f"Invalid ProcessingTask: {exc}",
|
|
25
|
+
) from exc
|
|
26
|
+
result: ProcessingResult = Engine().run(task)
|
|
27
|
+
return {"result": result.model_dump(mode="json")}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@router.get("/v1/engine/memory", dependencies=[Depends(require_api_key)])
|
|
31
|
+
async def engine_memory() -> Dict[str, Any]:
|
|
32
|
+
from ..engine.memory_utils import get_process_rss_mb, mps_available, torch_available
|
|
33
|
+
from ..engine.model_cache import get_model_cache
|
|
34
|
+
|
|
35
|
+
cache = get_model_cache()
|
|
36
|
+
return {
|
|
37
|
+
"rss_mb": get_process_rss_mb(),
|
|
38
|
+
"resident_slots": cache.resident_slots(),
|
|
39
|
+
"max_resident": cache.max_resident,
|
|
40
|
+
"evictions_total": cache.evictions_total,
|
|
41
|
+
"torch_available": torch_available(),
|
|
42
|
+
"mps_available": mps_available(),
|
|
43
|
+
}
|
|
@@ -94,7 +94,7 @@ async def _backfill_browser_visits_url_classification(
|
|
|
94
94
|
)
|
|
95
95
|
from ..sources.registry import BROWSER_VISITS
|
|
96
96
|
|
|
97
|
-
logger.
|
|
97
|
+
logger.debug(
|
|
98
98
|
"[PIPELINE:SIGNAL_DERIVE] Browser visits backfill from activity_events only_missing=%s limit=%s",
|
|
99
99
|
only_missing,
|
|
100
100
|
limit,
|
|
@@ -175,7 +175,7 @@ async def _backfill_browser_visits_url_classification(
|
|
|
175
175
|
"rows_failed": failed,
|
|
176
176
|
"errors": (derive_result.get("errors") or [])[:100],
|
|
177
177
|
}
|
|
178
|
-
logger.
|
|
178
|
+
logger.debug(
|
|
179
179
|
"[PIPELINE:SIGNAL_DERIVE] Browser visits backfill complete: scanned=%d processed=%d skipped=%d failed=%d",
|
|
180
180
|
summary["rows_scanned"],
|
|
181
181
|
summary["rows_processed"],
|
|
@@ -549,7 +549,7 @@ async def _process_enrichment_core(
|
|
|
549
549
|
tables_manager = DerivedTablesManager(conn=db_conn)
|
|
550
550
|
orchestrator = EnrichmentOrchestrator(tables_manager=tables_manager)
|
|
551
551
|
|
|
552
|
-
logger.
|
|
552
|
+
logger.debug(
|
|
553
553
|
"[PIPELINE:ENRICHMENT] %s: Manual enrichment triggered: source_id=%s, messages=%d, jobs=%s",
|
|
554
554
|
orchestrator,
|
|
555
555
|
source_id,
|
|
@@ -15,6 +15,7 @@ from .api import (
|
|
|
15
15
|
app_registry as app_registry_routes,
|
|
16
16
|
backup as backup_routes,
|
|
17
17
|
compute_remote as compute_remote_routes,
|
|
18
|
+
engine_run as engine_run_routes,
|
|
18
19
|
data_commit as data_commit_routes,
|
|
19
20
|
db as db_routes,
|
|
20
21
|
device as device_routes,
|
|
@@ -119,6 +120,7 @@ app.include_router(filter_lab_routes.router)
|
|
|
119
120
|
app.include_router(home_chat_routes.router)
|
|
120
121
|
app.include_router(privacy_disclose_routes.router)
|
|
121
122
|
app.include_router(compute_remote_routes.router)
|
|
123
|
+
app.include_router(engine_run_routes.router)
|
|
122
124
|
app.include_router(data_commit_routes.router)
|
|
123
125
|
|
|
124
126
|
|
|
@@ -5,9 +5,12 @@ import os
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import List, Optional
|
|
7
7
|
|
|
8
|
-
from pydantic import Field, model_validator
|
|
8
|
+
from pydantic import AliasChoices, Field, model_validator
|
|
9
9
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
10
10
|
|
|
11
|
+
# Set before any huggingface_hub import during app/route loading.
|
|
12
|
+
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
|
13
|
+
|
|
11
14
|
|
|
12
15
|
DEFAULT_TOPOS_CONTROL_PLANE_URL = "wss://cp.logu3s.com/ws/engine"
|
|
13
16
|
|
|
@@ -31,8 +34,14 @@ class Settings(BaseSettings):
|
|
|
31
34
|
|
|
32
35
|
openai_base_url: str = Field("https://api.openai.com/v1", env="OPENAI_BASE_URL")
|
|
33
36
|
openai_model: str = Field("gpt-4o-mini", env="OPENAI_MODEL")
|
|
34
|
-
red_pill_api_key: Optional[str] = Field(
|
|
35
|
-
|
|
37
|
+
red_pill_api_key: Optional[str] = Field(
|
|
38
|
+
None,
|
|
39
|
+
validation_alias=AliasChoices("RED_PILL_API_KEY", "REDPILL_API_KEY"),
|
|
40
|
+
)
|
|
41
|
+
red_pill_api_base: str = Field(
|
|
42
|
+
"https://api.redpill.ai/v1",
|
|
43
|
+
validation_alias=AliasChoices("RED_PILL_API_BASE", "REDPILL_API_BASE"),
|
|
44
|
+
)
|
|
36
45
|
|
|
37
46
|
gt_cloud_api_key: Optional[str] = Field(None, env="GT_CLOUD_API_KEY")
|
|
38
47
|
griptape_nodes_api_base_url: str = Field(
|
|
@@ -92,6 +101,12 @@ class Settings(BaseSettings):
|
|
|
92
101
|
ollama_query_model: str = Field("llama3.2:latest", env="TOPOS_OLLAMA_QUERY_MODEL")
|
|
93
102
|
engine_default_provider: str = Field("huggingface", env="ENGINE_DEFAULT_PROVIDER")
|
|
94
103
|
|
|
104
|
+
engine_max_resident_models: int = Field(3, env="ENGINE_MAX_RESIDENT_MODELS")
|
|
105
|
+
engine_model_idle_ttl_sec: int = Field(0, env="ENGINE_MODEL_IDLE_TTL_SEC")
|
|
106
|
+
engine_memory_rss_soft_limit_mb: int = Field(0, env="ENGINE_MEMORY_RSS_SOFT_LIMIT_MB")
|
|
107
|
+
engine_flush_after_task: bool = Field(False, env="ENGINE_FLUSH_AFTER_TASK")
|
|
108
|
+
engine_ml_device: Optional[str] = Field(None, env="ENGINE_ML_DEVICE")
|
|
109
|
+
|
|
95
110
|
# Sanitization field-transforms via Ollama (see topos.config.sanitization_ollama + engine_config overrides)
|
|
96
111
|
sanitization_ollama_enabled: bool = Field(False, env="SANITIZATION_OLLAMA_ENABLED")
|
|
97
112
|
sanitization_ollama_host: Optional[str] = Field(None, env="SANITIZATION_OLLAMA_HOST")
|
|
@@ -171,7 +171,7 @@ def _uma_transform_progress_hook(req_id: str, stage_label: str):
|
|
|
171
171
|
return
|
|
172
172
|
state["bucket"] = bucket
|
|
173
173
|
filter_text = f" filter={current_filter}" if current_filter else ""
|
|
174
|
-
logger.
|
|
174
|
+
logger.debug(
|
|
175
175
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=%s%s %s %s%% (%s/%s)",
|
|
176
176
|
req_id,
|
|
177
177
|
stage_label,
|
|
@@ -2118,7 +2118,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
2118
2118
|
if not existing_user_id:
|
|
2119
2119
|
# Engine started but no user_id yet - link to authenticated user
|
|
2120
2120
|
store_user_id(conn, auth_user_id)
|
|
2121
|
-
logger.
|
|
2121
|
+
logger.debug(
|
|
2122
2122
|
"[PIPELINE:CONNECTION] Engine had no user_id. Stored auth user_id=%s from control plane. "
|
|
2123
2123
|
"Engine is now linked to authenticated user.",
|
|
2124
2124
|
auth_user_id[:8] if auth_user_id else None,
|
|
@@ -4178,7 +4178,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
4178
4178
|
if not enrichment_name:
|
|
4179
4179
|
return {"id": req_id, "status": "error", "error": "enrichment_name required"}
|
|
4180
4180
|
try:
|
|
4181
|
-
logger.
|
|
4181
|
+
logger.debug(
|
|
4182
4182
|
"[PIPELINE:ENRICHMENT] source_enrichment_backfill received: source_id=%s enrichment=%s only_missing=%s limit=%s",
|
|
4183
4183
|
source_id,
|
|
4184
4184
|
enrichment_name,
|
|
@@ -4193,7 +4193,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
4193
4193
|
only_missing=bool(only_missing),
|
|
4194
4194
|
limit=limit,
|
|
4195
4195
|
)
|
|
4196
|
-
logger.
|
|
4196
|
+
logger.debug(
|
|
4197
4197
|
"[PIPELINE:ENRICHMENT] source_enrichment_backfill complete: source_id=%s enrichment=%s rows_scanned=%s rows_processed=%s rows_failed=%s",
|
|
4198
4198
|
source_id,
|
|
4199
4199
|
enrichment_name,
|
|
@@ -4215,7 +4215,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
4215
4215
|
if not enrichment_name:
|
|
4216
4216
|
return {"id": req_id, "status": "error", "error": "enrichment_name required"}
|
|
4217
4217
|
try:
|
|
4218
|
-
logger.
|
|
4218
|
+
logger.debug(
|
|
4219
4219
|
"[PIPELINE:ENRICHMENT] source_enrichment_test received: source_id=%s enrichment=%s",
|
|
4220
4220
|
source_id,
|
|
4221
4221
|
enrichment_name,
|
|
@@ -5359,7 +5359,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5359
5359
|
requesting_app_id = (payload.get("requesting_app_id") or "").strip() or None
|
|
5360
5360
|
requesting_app_name = (payload.get("requesting_app_name") or "").strip() or None
|
|
5361
5361
|
app_display = requesting_app_name or requesting_app_id or "(unknown app)"
|
|
5362
|
-
logger.
|
|
5362
|
+
logger.debug(
|
|
5363
5363
|
"[PIPELINE:UMA] uma_get_messages: message_stream=%s, resource_id=%s, dataset_id=%s, limit=%s, offset=%s, requesting_user_email=%s, requesting_app=%s",
|
|
5364
5364
|
message_stream,
|
|
5365
5365
|
resource_id[:50] if resource_id else None,
|
|
@@ -5502,12 +5502,12 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5502
5502
|
pre_name_rows = sum(1 for row in after_contact if row.get("sender_display_name"))
|
|
5503
5503
|
manifest_for_generic = strip_contact_runtime_filters(filter_manifest)
|
|
5504
5504
|
transform_diag: Dict[str, Any] = {}
|
|
5505
|
-
logger.
|
|
5505
|
+
logger.debug(
|
|
5506
5506
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=conversation_messages start rows=%s",
|
|
5507
5507
|
req_id,
|
|
5508
5508
|
len(after_contact),
|
|
5509
5509
|
)
|
|
5510
|
-
logger.
|
|
5510
|
+
logger.debug(
|
|
5511
5511
|
"[PIPELINE:UMA] contact names: contacts_resolve=%s names_effective=%s "
|
|
5512
5512
|
"sender_display_name_rows_after_pipeline=%s allowed_scopes=%s",
|
|
5513
5513
|
contacts_resolve,
|
|
@@ -5523,7 +5523,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5523
5523
|
diagnostics=transform_diag,
|
|
5524
5524
|
progress_hook=_uma_transform_progress_hook(req_id, "conversation_messages"),
|
|
5525
5525
|
)
|
|
5526
|
-
logger.
|
|
5526
|
+
logger.debug(
|
|
5527
5527
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=conversation_messages done applied=%s skipped=%s reasons=%s",
|
|
5528
5528
|
req_id,
|
|
5529
5529
|
transform_diag.get("applied_count", 0),
|
|
@@ -5532,7 +5532,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5532
5532
|
)
|
|
5533
5533
|
_skip_reasons = transform_diag.get("skip_reasons") or {}
|
|
5534
5534
|
if _skip_reasons.get("table_mismatch"):
|
|
5535
|
-
logger.
|
|
5535
|
+
logger.debug(
|
|
5536
5536
|
"[PIPELINE:UMA] req=%s table_mismatch skips=%s are expected when field_transforms "
|
|
5537
5537
|
"include other tables (browser_visits, etc.) while processing conversation_messages.",
|
|
5538
5538
|
req_id,
|
|
@@ -5707,7 +5707,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5707
5707
|
pre_contact_len,
|
|
5708
5708
|
)
|
|
5709
5709
|
transform_diag = {}
|
|
5710
|
-
logger.
|
|
5710
|
+
logger.debug(
|
|
5711
5711
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=ai_chat_messages start rows=%s",
|
|
5712
5712
|
req_id,
|
|
5713
5713
|
len(messages),
|
|
@@ -5720,7 +5720,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5720
5720
|
diagnostics=transform_diag,
|
|
5721
5721
|
progress_hook=_uma_transform_progress_hook(req_id, "ai_chat_messages"),
|
|
5722
5722
|
)
|
|
5723
|
-
logger.
|
|
5723
|
+
logger.debug(
|
|
5724
5724
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=ai_chat_messages done applied=%s skipped=%s reasons=%s",
|
|
5725
5725
|
req_id,
|
|
5726
5726
|
transform_diag.get("applied_count", 0),
|
|
@@ -5764,7 +5764,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5764
5764
|
pre_contact_len,
|
|
5765
5765
|
)
|
|
5766
5766
|
transform_diag = {}
|
|
5767
|
-
logger.
|
|
5767
|
+
logger.debug(
|
|
5768
5768
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=ai_chat_messages_jsonl start rows=%s",
|
|
5769
5769
|
req_id,
|
|
5770
5770
|
len(messages),
|
|
@@ -5777,7 +5777,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5777
5777
|
diagnostics=transform_diag,
|
|
5778
5778
|
progress_hook=_uma_transform_progress_hook(req_id, "ai_chat_messages_jsonl"),
|
|
5779
5779
|
)
|
|
5780
|
-
logger.
|
|
5780
|
+
logger.debug(
|
|
5781
5781
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=ai_chat_messages_jsonl done applied=%s skipped=%s reasons=%s",
|
|
5782
5782
|
req_id,
|
|
5783
5783
|
transform_diag.get("applied_count", 0),
|
|
@@ -5951,7 +5951,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5951
5951
|
rows = all_rows[:limit]
|
|
5952
5952
|
try:
|
|
5953
5953
|
transform_diag = {}
|
|
5954
|
-
logger.
|
|
5954
|
+
logger.debug(
|
|
5955
5955
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=rows:%s start rows=%s",
|
|
5956
5956
|
req_id,
|
|
5957
5957
|
table_name,
|
|
@@ -5965,7 +5965,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
5965
5965
|
diagnostics=transform_diag,
|
|
5966
5966
|
progress_hook=_uma_transform_progress_hook(req_id, f"rows:{table_name}"),
|
|
5967
5967
|
)
|
|
5968
|
-
logger.
|
|
5968
|
+
logger.debug(
|
|
5969
5969
|
"[PIPELINE:UMA][TRANSFORM] req=%s stage=rows:%s done applied=%s skipped=%s reasons=%s",
|
|
5970
5970
|
req_id,
|
|
5971
5971
|
table_name,
|
|
@@ -6840,7 +6840,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
6840
6840
|
sync_options = payload.get("sync_options")
|
|
6841
6841
|
if source_id == "imessage" and not sync_options:
|
|
6842
6842
|
sync_options = {"mode": "3m"}
|
|
6843
|
-
logger.
|
|
6843
|
+
logger.debug(
|
|
6844
6844
|
"[PIPELINE:SYNC] Engine received source_sync request: source_id=%s dataset_id=%s sync_options=%s (req_id=%s)",
|
|
6845
6845
|
source_id,
|
|
6846
6846
|
dataset_id[:24] + "..." if dataset_id and len(dataset_id) > 24 else dataset_id,
|
|
@@ -6870,7 +6870,7 @@ async def handle_control_plane_request(message: Dict[str, Any]) -> Optional[Dict
|
|
|
6870
6870
|
error,
|
|
6871
6871
|
exc_info=False,
|
|
6872
6872
|
)
|
|
6873
|
-
logger.
|
|
6873
|
+
logger.debug("[PIPELINE:SYNC] source_sync completed: source_id=%s status=%s", source_id, status)
|
|
6874
6874
|
if conn and status == "ok":
|
|
6875
6875
|
update_sync_result(conn, dataset_id, source_id, success=True, last_sync_at=datetime.now(timezone.utc).isoformat())
|
|
6876
6876
|
elif conn and status == "error":
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
3
5
|
import sys
|
|
4
6
|
from datetime import datetime
|
|
5
7
|
from time import time
|
|
@@ -9,6 +11,26 @@ from ..runtime_update import is_update_available
|
|
|
9
11
|
|
|
10
12
|
_LOG_FORMAT: str | None = None
|
|
11
13
|
|
|
14
|
+
# Before huggingface_hub/transformers import, suppress download/load tqdm bars.
|
|
15
|
+
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def suppress_ml_progress_bars() -> None:
|
|
19
|
+
"""Hide Hugging Face / transformers tqdm progress bars; keep app INFO load logs."""
|
|
20
|
+
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
|
21
|
+
try:
|
|
22
|
+
from huggingface_hub.utils import disable_progress_bars
|
|
23
|
+
|
|
24
|
+
disable_progress_bars()
|
|
25
|
+
except Exception:
|
|
26
|
+
pass
|
|
27
|
+
try:
|
|
28
|
+
from transformers.utils.logging import disable_progress_bar
|
|
29
|
+
|
|
30
|
+
disable_progress_bar()
|
|
31
|
+
except Exception:
|
|
32
|
+
pass
|
|
33
|
+
|
|
12
34
|
# Route uvicorn loggers through the root handler configured by configure_logging().
|
|
13
35
|
UVICORN_LOG_CONFIG: dict[str, object] = {
|
|
14
36
|
"version": 1,
|
|
@@ -37,6 +59,7 @@ def align_uvicorn_loggers() -> None:
|
|
|
37
59
|
|
|
38
60
|
def configure_logging() -> None:
|
|
39
61
|
"""Configure logging to stdout based on environment/log settings."""
|
|
62
|
+
suppress_ml_progress_bars()
|
|
40
63
|
handler = logging.StreamHandler(sys.stdout)
|
|
41
64
|
|
|
42
65
|
def resolve_format() -> str:
|
|
@@ -145,12 +168,42 @@ class ColorFormatter(logging.Formatter):
|
|
|
145
168
|
|
|
146
169
|
# Message text: Same blue as DEBUG
|
|
147
170
|
_MESSAGE_COLOR = "\x1b[38;5;26m" # Light blue (same as DEBUG)
|
|
171
|
+
|
|
172
|
+
# source / source_id values: bold magenta — easy to scan in pipeline logs
|
|
173
|
+
_SOURCE_IDENTITY_VALUE_COLOR = "\x1b[1;38;5;213m" # Bold bright magenta
|
|
148
174
|
|
|
149
175
|
# Logger name (module path): Same teal/cyan as class names
|
|
150
176
|
_LOGGER_NAME_COLOR = "\x1b[38;5;75m" # Teal/cyan blue-green
|
|
151
177
|
|
|
152
178
|
_ARROW_COLOR = "\x1b[38;5;244m" # Gray for arrows
|
|
153
179
|
|
|
180
|
+
_SOURCE_KV_PATTERN = re.compile(
|
|
181
|
+
r"(?<![?&/])" # skip URL query/path fragments like ...&source=editors
|
|
182
|
+
r"\b(?P<key>source_id|source)="
|
|
183
|
+
r"(?:"
|
|
184
|
+
r"(?P<q>['\"])(?P<qval>[^'\"]+)(?P=q)|"
|
|
185
|
+
r"(?P<uval>[^\s,;)\]&]+)"
|
|
186
|
+
r")"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
_SOURCE_VALUE_SKIP = frozenset({"%s", "%d", "%i", "%f", "?"})
|
|
190
|
+
|
|
191
|
+
def _highlight_source_identities(self, message: str) -> str:
|
|
192
|
+
"""Bold magenta on source / source_id values (keys stay default message color)."""
|
|
193
|
+
|
|
194
|
+
def repl(match: re.Match[str]) -> str:
|
|
195
|
+
value = match.group("qval") or match.group("uval") or ""
|
|
196
|
+
if value in self._SOURCE_VALUE_SKIP or value.startswith("excluded."):
|
|
197
|
+
return match.group(0)
|
|
198
|
+
key = match.group("key")
|
|
199
|
+
quote = match.group("q") or ""
|
|
200
|
+
colored_value = (
|
|
201
|
+
f"{self._SOURCE_IDENTITY_VALUE_COLOR}{value}{self._RESET}{self._MESSAGE_COLOR}"
|
|
202
|
+
)
|
|
203
|
+
return f"{key}={quote}{colored_value}{quote}"
|
|
204
|
+
|
|
205
|
+
return self._SOURCE_KV_PATTERN.sub(repl, message)
|
|
206
|
+
|
|
154
207
|
def format(self, record: logging.LogRecord) -> str: # noqa: D401
|
|
155
208
|
# Format timestamp: YYYY-MM-DD HH:MM:SS.ms (green)
|
|
156
209
|
timestamp_str = datetime.fromtimestamp(record.created).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
|
|
@@ -171,8 +224,7 @@ class ColorFormatter(logging.Formatter):
|
|
|
171
224
|
|
|
172
225
|
# Get message and extract pipeline tag if present
|
|
173
226
|
message = record.getMessage()
|
|
174
|
-
|
|
175
|
-
|
|
227
|
+
|
|
176
228
|
# Extract pipeline stage tag from message if it exists
|
|
177
229
|
pipeline_tag_pattern = r'\[PIPELINE:[A-Z_]+\]'
|
|
178
230
|
pipeline_tag_match = re.search(pipeline_tag_pattern, message)
|
|
@@ -204,6 +256,8 @@ class ColorFormatter(logging.Formatter):
|
|
|
204
256
|
f"{self._ARROW_COLOR}\\1{self._RESET}{self._MESSAGE_COLOR}",
|
|
205
257
|
message_colored
|
|
206
258
|
)
|
|
259
|
+
|
|
260
|
+
message_colored = self._highlight_source_identities(message_colored)
|
|
207
261
|
|
|
208
262
|
# Format: TIMESTAMP | LEVEL | [PIPELINE:TAG] | LOGGER: MESSAGE
|
|
209
263
|
separator = f"{self._SEPARATOR_COLOR}|{self._RESET}"
|
|
@@ -69,13 +69,13 @@ class PrivacyLayerClient:
|
|
|
69
69
|
*,
|
|
70
70
|
transform_id: str,
|
|
71
71
|
) -> Dict[str, Any]:
|
|
72
|
-
from ..engine import
|
|
72
|
+
from ..engine.client import get_engine_client_or_local
|
|
73
73
|
from ..enrichment.jobs.canonical._engine_runner import run_engine_task
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
client = get_engine_client_or_local(self._engine)
|
|
76
76
|
record_ids = [str(i.get("id") or "") for i in items]
|
|
77
77
|
result = await run_engine_task(
|
|
78
|
-
|
|
78
|
+
client,
|
|
79
79
|
task_id=f"privacy_{record_ids[0] or 'batch'}",
|
|
80
80
|
subtype="privacy_disclosure",
|
|
81
81
|
source_id=None,
|
|
@@ -127,15 +127,15 @@ class PrivacyLayerClient:
|
|
|
127
127
|
return await self._classify_nsfw_via_engine(items)
|
|
128
128
|
|
|
129
129
|
async def _classify_nsfw_via_engine(self, items: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
130
|
-
from ..engine import Engine
|
|
131
|
-
from ..enrichment.jobs.canonical._engine_runner import run_engine_task
|
|
132
130
|
from ..config.settings import settings
|
|
131
|
+
from ..engine.client import get_engine_client_or_local
|
|
132
|
+
from ..enrichment.jobs.canonical._engine_runner import run_engine_task
|
|
133
133
|
|
|
134
|
-
|
|
134
|
+
client = get_engine_client_or_local(self._engine)
|
|
135
135
|
model = getattr(settings, "nsfw_classifier_model", "michellejieli/NSFW_text_classifier")
|
|
136
136
|
record_ids = [str(i.get("id") or "") for i in items]
|
|
137
137
|
result = await run_engine_task(
|
|
138
|
-
|
|
138
|
+
client,
|
|
139
139
|
task_id=f"nsfw_{record_ids[0] or 'batch'}",
|
|
140
140
|
subtype="content_nsfw_classification",
|
|
141
141
|
source_id=None,
|
|
@@ -184,11 +184,11 @@ async def run_privacy_disclosure_layer(
|
|
|
184
184
|
apply_canonical_nsfw_v1_up(conn)
|
|
185
185
|
|
|
186
186
|
if not getattr(settings, "platform_privacy_via_engine", True):
|
|
187
|
-
logger.
|
|
187
|
+
logger.debug("[PIPELINE:PRIVACY] skipped: platform_privacy_via_engine=false")
|
|
188
188
|
return {"records_updated": 0, "skipped": True}
|
|
189
189
|
|
|
190
190
|
if nsfw_only and not getattr(settings, "nsfw_classifier_enabled", True):
|
|
191
|
-
logger.
|
|
191
|
+
logger.debug("[PIPELINE:PRIVACY] nsfw-only backfill skipped: nsfw_classifier_enabled=false")
|
|
192
192
|
return {"records_updated": 0, "nsfw_tagged": 0, "skipped": True}
|
|
193
193
|
|
|
194
194
|
if not canonical_messages:
|
|
@@ -353,7 +353,7 @@ async def run_privacy_disclosure_layer(
|
|
|
353
353
|
conn.commit()
|
|
354
354
|
|
|
355
355
|
duration_ms = int((time.perf_counter() - started) * 1000)
|
|
356
|
-
logger.
|
|
356
|
+
logger.debug(
|
|
357
357
|
"[PIPELINE:PRIVACY] platform_privacy_layer disclosure_updated=%d nsfw_tagged=%d "
|
|
358
358
|
"failed_batches=%d nsfw_failed_batches=%d duration_ms=%d version=%s",
|
|
359
359
|
updated,
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""Topos Engine: unified runtime for ML/LLM processing (enrichments, transformations, queries)."""
|
|
2
2
|
|
|
3
3
|
from .engine import Engine
|
|
4
|
+
from .client import EngineClient, LocalEngineClient, RemoteEngineClient, get_engine_client, get_engine_client_or_local
|
|
5
|
+
from .model_cache import ModelSlot, get_model_cache
|
|
4
6
|
from .tasks import (
|
|
5
7
|
ExecutionMeta,
|
|
6
8
|
ExecutionSpec,
|
|
@@ -16,6 +18,13 @@ from .tasks import (
|
|
|
16
18
|
|
|
17
19
|
__all__ = [
|
|
18
20
|
"Engine",
|
|
21
|
+
"EngineClient",
|
|
22
|
+
"LocalEngineClient",
|
|
23
|
+
"RemoteEngineClient",
|
|
24
|
+
"get_engine_client",
|
|
25
|
+
"get_engine_client_or_local",
|
|
26
|
+
"ModelSlot",
|
|
27
|
+
"get_model_cache",
|
|
19
28
|
"ProcessingTask",
|
|
20
29
|
"ProcessingResult",
|
|
21
30
|
"ModelRequest",
|