bizon 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bizon/alerting/alerts.py +0 -1
- bizon/common/models.py +182 -4
- bizon/connectors/destinations/bigquery/src/config.py +0 -1
- bizon/connectors/destinations/bigquery/src/destination.py +11 -8
- bizon/connectors/destinations/bigquery_streaming/config/bigquery_streaming.example.yml +74 -0
- bizon/connectors/destinations/bigquery_streaming/src/destination.py +4 -5
- bizon/connectors/destinations/bigquery_streaming_v2/config/bigquery_streaming_v2.example.yml +79 -0
- bizon/connectors/destinations/bigquery_streaming_v2/src/destination.py +4 -6
- bizon/connectors/destinations/file/config/file.example.yml +40 -0
- bizon/connectors/destinations/file/src/config.py +1 -1
- bizon/connectors/destinations/file/src/destination.py +0 -5
- bizon/connectors/destinations/logger/config/logger.example.yml +30 -0
- bizon/connectors/destinations/logger/src/config.py +0 -2
- bizon/connectors/destinations/logger/src/destination.py +1 -2
- bizon/connectors/sources/cycle/src/source.py +2 -6
- bizon/connectors/sources/dummy/src/source.py +0 -4
- bizon/connectors/sources/gsheets/src/source.py +2 -3
- bizon/connectors/sources/hubspot/src/hubspot_base.py +0 -1
- bizon/connectors/sources/hubspot/src/hubspot_objects.py +3 -4
- bizon/connectors/sources/hubspot/src/models/hs_object.py +0 -1
- bizon/connectors/sources/kafka/config/kafka_streams.example.yml +124 -0
- bizon/connectors/sources/kafka/src/config.py +10 -6
- bizon/connectors/sources/kafka/src/decode.py +2 -2
- bizon/connectors/sources/kafka/src/source.py +147 -46
- bizon/connectors/sources/notion/config/api_key.example.yml +35 -0
- bizon/connectors/sources/notion/src/__init__.py +0 -0
- bizon/connectors/sources/notion/src/config.py +59 -0
- bizon/connectors/sources/notion/src/source.py +1159 -0
- bizon/connectors/sources/notion/tests/notion_pipeline.py +7 -0
- bizon/connectors/sources/notion/tests/test_notion.py +113 -0
- bizon/connectors/sources/periscope/src/source.py +0 -6
- bizon/connectors/sources/pokeapi/src/source.py +0 -1
- bizon/connectors/sources/sana_ai/config/sana.example.yml +25 -0
- bizon/connectors/sources/sana_ai/src/source.py +85 -0
- bizon/destination/buffer.py +0 -1
- bizon/destination/config.py +0 -1
- bizon/destination/destination.py +1 -4
- bizon/engine/backend/adapters/sqlalchemy/backend.py +2 -5
- bizon/engine/backend/adapters/sqlalchemy/config.py +0 -1
- bizon/engine/config.py +0 -1
- bizon/engine/engine.py +0 -1
- bizon/engine/pipeline/consumer.py +0 -1
- bizon/engine/pipeline/producer.py +1 -5
- bizon/engine/queue/adapters/kafka/config.py +1 -1
- bizon/engine/queue/adapters/kafka/queue.py +0 -1
- bizon/engine/queue/adapters/python_queue/consumer.py +0 -1
- bizon/engine/queue/adapters/python_queue/queue.py +0 -2
- bizon/engine/queue/adapters/rabbitmq/consumer.py +0 -1
- bizon/engine/queue/adapters/rabbitmq/queue.py +0 -1
- bizon/engine/queue/config.py +0 -2
- bizon/engine/runner/adapters/process.py +0 -2
- bizon/engine/runner/adapters/streaming.py +55 -1
- bizon/engine/runner/adapters/thread.py +0 -2
- bizon/engine/runner/config.py +0 -1
- bizon/engine/runner/runner.py +0 -2
- bizon/monitoring/datadog/monitor.py +5 -3
- bizon/monitoring/noop/monitor.py +1 -1
- bizon/source/auth/authenticators/abstract_oauth.py +11 -3
- bizon/source/auth/authenticators/abstract_token.py +2 -1
- bizon/source/auth/authenticators/basic.py +1 -1
- bizon/source/auth/authenticators/cookies.py +2 -1
- bizon/source/auth/authenticators/oauth.py +8 -3
- bizon/source/config.py +0 -2
- bizon/source/cursor.py +8 -16
- bizon/source/discover.py +3 -6
- bizon/source/models.py +0 -1
- bizon/source/session.py +0 -1
- bizon/source/source.py +17 -2
- bizon/transform/config.py +0 -2
- bizon/transform/transform.py +0 -3
- {bizon-0.1.2.dist-info → bizon-0.2.0.dist-info}/METADATA +62 -42
- bizon-0.2.0.dist-info/RECORD +136 -0
- {bizon-0.1.2.dist-info → bizon-0.2.0.dist-info}/WHEEL +1 -1
- bizon-0.2.0.dist-info/entry_points.txt +2 -0
- bizon-0.1.2.dist-info/RECORD +0 -123
- bizon-0.1.2.dist-info/entry_points.txt +0 -3
- {bizon-0.1.2.dist-info → bizon-0.2.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import Any, Dict, List
|
|
3
|
+
|
|
4
|
+
from pydantic import Field
|
|
5
|
+
|
|
6
|
+
from bizon.source.config import SourceConfig
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class NotionStreams(str, Enum):
|
|
10
|
+
DATABASES = "databases"
|
|
11
|
+
DATA_SOURCES = "data_sources"
|
|
12
|
+
PAGES = "pages"
|
|
13
|
+
BLOCKS = "blocks"
|
|
14
|
+
BLOCKS_MARKDOWN = "blocks_markdown"
|
|
15
|
+
USERS = "users"
|
|
16
|
+
# Streams that fetch all accessible content (no database_ids/page_ids required)
|
|
17
|
+
ALL_PAGES = "all_pages"
|
|
18
|
+
ALL_DATABASES = "all_databases"
|
|
19
|
+
ALL_DATA_SOURCES = "all_data_sources"
|
|
20
|
+
ALL_BLOCKS_MARKDOWN = "all_blocks_markdown"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NotionSourceConfig(SourceConfig):
|
|
24
|
+
stream: NotionStreams
|
|
25
|
+
|
|
26
|
+
database_ids: List[str] = Field(
|
|
27
|
+
default_factory=list,
|
|
28
|
+
description="List of Notion database IDs to fetch. Required for databases, data_sources, pages, and blocks streams.",
|
|
29
|
+
)
|
|
30
|
+
page_ids: List[str] = Field(
|
|
31
|
+
default_factory=list,
|
|
32
|
+
description="List of Notion page IDs to fetch. Used for pages and blocks streams.",
|
|
33
|
+
)
|
|
34
|
+
fetch_blocks_recursively: bool = Field(
|
|
35
|
+
default=True,
|
|
36
|
+
description="Whether to fetch nested blocks recursively. Only applies to blocks stream.",
|
|
37
|
+
)
|
|
38
|
+
max_recursion_depth: int = Field(
|
|
39
|
+
default=5,
|
|
40
|
+
ge=1,
|
|
41
|
+
le=100,
|
|
42
|
+
description="Maximum nesting depth for recursive block fetching. Prevents infinite loops.",
|
|
43
|
+
)
|
|
44
|
+
page_size: int = Field(
|
|
45
|
+
default=100,
|
|
46
|
+
ge=1,
|
|
47
|
+
le=100,
|
|
48
|
+
description="Number of results per page (max 100)",
|
|
49
|
+
)
|
|
50
|
+
max_workers: int = Field(
|
|
51
|
+
default=3,
|
|
52
|
+
ge=1,
|
|
53
|
+
le=10,
|
|
54
|
+
description="Number of concurrent workers for fetching blocks. Keep low to respect rate limits.",
|
|
55
|
+
)
|
|
56
|
+
database_filters: Dict[str, Any] = Field(
|
|
57
|
+
default_factory=dict,
|
|
58
|
+
description="Map of database_id -> Notion filter object. Filters are passed directly to Notion API.",
|
|
59
|
+
)
|