nia-sync 0.1.5__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nia_sync-0.1.5 → nia_sync-0.1.7}/PKG-INFO +1 -1
- {nia_sync-0.1.5 → nia_sync-0.1.7}/nia_sync.egg-info/PKG-INFO +1 -1
- {nia_sync-0.1.5 → nia_sync-0.1.7}/pyproject.toml +1 -1
- {nia_sync-0.1.5 → nia_sync-0.1.7}/sync.py +59 -3
- {nia_sync-0.1.5 → nia_sync-0.1.7}/auth.py +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/config.py +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/extractor.py +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/main.py +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/nia_sync.egg-info/SOURCES.txt +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/nia_sync.egg-info/dependency_links.txt +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/nia_sync.egg-info/entry_points.txt +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/nia_sync.egg-info/requires.txt +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/nia_sync.egg-info/top_level.txt +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/setup.cfg +0 -0
- {nia_sync-0.1.5 → nia_sync-0.1.7}/watcher.py +0 -0
|
@@ -15,7 +15,7 @@ from typing import Any
|
|
|
15
15
|
import httpx
|
|
16
16
|
|
|
17
17
|
from config import API_BASE_URL, get_api_key
|
|
18
|
-
from extractor import extract_incremental, detect_source_type
|
|
18
|
+
from extractor import extract_incremental, detect_source_type, TYPE_FOLDER
|
|
19
19
|
|
|
20
20
|
logger = logging.getLogger(__name__)
|
|
21
21
|
|
|
@@ -26,10 +26,31 @@ MAX_BATCH_SIZE_BYTES = 4 * 1024 * 1024 # 4MB max payload per batch
|
|
|
26
26
|
MAX_RETRIES = 4
|
|
27
27
|
RETRY_BASE_DELAY = 1.5
|
|
28
28
|
RETRY_MAX_DELAY = 15.0
|
|
29
|
+
FOLDER_CURSOR_VERSION = 1
|
|
29
30
|
|
|
30
31
|
# Reusable client for connection pooling
|
|
31
32
|
_http_client: httpx.Client | None = None
|
|
32
33
|
|
|
34
|
+
def _normalize_folder_cursor(
|
|
35
|
+
path: str,
|
|
36
|
+
cursor: dict[str, Any] | None,
|
|
37
|
+
) -> tuple[dict[str, Any], str | None]:
|
|
38
|
+
if not cursor:
|
|
39
|
+
return {}, "missing"
|
|
40
|
+
|
|
41
|
+
cursor_version = cursor.get("cursor_version")
|
|
42
|
+
root_path = cursor.get("root_path")
|
|
43
|
+
normalized_root = os.path.abspath(os.path.expanduser(root_path)) if root_path else None
|
|
44
|
+
|
|
45
|
+
if cursor_version != FOLDER_CURSOR_VERSION:
|
|
46
|
+
return {}, "version_mismatch"
|
|
47
|
+
if not normalized_root:
|
|
48
|
+
return {}, "missing_root_path"
|
|
49
|
+
if normalized_root != path:
|
|
50
|
+
return {}, "root_path_changed"
|
|
51
|
+
|
|
52
|
+
return cursor, None
|
|
53
|
+
|
|
33
54
|
def get_http_client() -> httpx.Client:
|
|
34
55
|
"""Get or create HTTP client with connection pooling."""
|
|
35
56
|
global _http_client
|
|
@@ -88,8 +109,9 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
|
|
|
88
109
|
"message": "No local path configured",
|
|
89
110
|
}
|
|
90
111
|
|
|
91
|
-
# Expand ~
|
|
112
|
+
# Expand ~ and normalize path
|
|
92
113
|
path = os.path.expanduser(path)
|
|
114
|
+
path = os.path.abspath(path)
|
|
93
115
|
|
|
94
116
|
# Validate path exists
|
|
95
117
|
if not os.path.exists(path):
|
|
@@ -105,6 +127,12 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
|
|
|
105
127
|
if not detected_type:
|
|
106
128
|
detected_type = detect_source_type(path)
|
|
107
129
|
|
|
130
|
+
cursor_reset_reason = None
|
|
131
|
+
if detected_type == TYPE_FOLDER:
|
|
132
|
+
cursor, cursor_reset_reason = _normalize_folder_cursor(path, cursor)
|
|
133
|
+
if cursor_reset_reason:
|
|
134
|
+
logger.info(f"Resetting folder sync cursor for {path} ({cursor_reset_reason})")
|
|
135
|
+
|
|
108
136
|
logger.info(f"Syncing {path} (type={detected_type})")
|
|
109
137
|
|
|
110
138
|
try:
|
|
@@ -116,10 +144,38 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
|
|
|
116
144
|
)
|
|
117
145
|
|
|
118
146
|
files = extraction_result.get("files", [])
|
|
119
|
-
new_cursor = extraction_result.get("cursor", {})
|
|
147
|
+
new_cursor = dict(extraction_result.get("cursor", {}))
|
|
120
148
|
stats = extraction_result.get("stats", {})
|
|
121
149
|
|
|
150
|
+
if detected_type == TYPE_FOLDER:
|
|
151
|
+
new_cursor["cursor_version"] = FOLDER_CURSOR_VERSION
|
|
152
|
+
new_cursor["root_path"] = path
|
|
153
|
+
|
|
122
154
|
if not files:
|
|
155
|
+
if detected_type == TYPE_FOLDER and cursor_reset_reason:
|
|
156
|
+
cursor_update = new_cursor
|
|
157
|
+
upload_result = upload_sync_data(
|
|
158
|
+
local_folder_id=local_folder_id,
|
|
159
|
+
files=[],
|
|
160
|
+
cursor=cursor_update,
|
|
161
|
+
stats=stats,
|
|
162
|
+
is_final_batch=True,
|
|
163
|
+
)
|
|
164
|
+
if upload_result.get("status") == "ok":
|
|
165
|
+
source["cursor"] = cursor_update
|
|
166
|
+
return {
|
|
167
|
+
"path": path,
|
|
168
|
+
"status": "success",
|
|
169
|
+
"added": 0,
|
|
170
|
+
"message": "No new data (cursor updated)",
|
|
171
|
+
}
|
|
172
|
+
report_sync_error(local_folder_id, upload_result.get("message", "Upload failed"), path)
|
|
173
|
+
return {
|
|
174
|
+
"path": path,
|
|
175
|
+
"status": "error",
|
|
176
|
+
"error": upload_result.get("message", "Upload failed"),
|
|
177
|
+
}
|
|
178
|
+
|
|
123
179
|
logger.info(f"No new data to sync for {path}")
|
|
124
180
|
return {
|
|
125
181
|
"path": path,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|