nia-sync 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- config.py +4 -1
- extractor.py +2 -2
- main.py +49 -9
- {nia_sync-0.1.2.dist-info → nia_sync-0.1.4.dist-info}/METADATA +1 -1
- nia_sync-0.1.4.dist-info/RECORD +11 -0
- {nia_sync-0.1.2.dist-info → nia_sync-0.1.4.dist-info}/WHEEL +1 -1
- sync.py +33 -5
- nia_sync-0.1.2.dist-info/RECORD +0 -11
- {nia_sync-0.1.2.dist-info → nia_sync-0.1.4.dist-info}/entry_points.txt +0 -0
- {nia_sync-0.1.2.dist-info → nia_sync-0.1.4.dist-info}/top_level.txt +0 -0
config.py
CHANGED
|
@@ -133,7 +133,10 @@ def clear_config():
|
|
|
133
133
|
|
|
134
134
|
|
|
135
135
|
def get_api_key() -> str | None:
|
|
136
|
-
"""Get the stored API key."""
|
|
136
|
+
"""Get the stored API key. Checks NIA_API_KEY env var first, then config file."""
|
|
137
|
+
env_key = os.getenv("NIA_API_KEY")
|
|
138
|
+
if env_key:
|
|
139
|
+
return env_key
|
|
137
140
|
config = load_config()
|
|
138
141
|
return config.get("api_key")
|
|
139
142
|
|
extractor.py
CHANGED
|
@@ -64,8 +64,8 @@ SKIP_DIRS = {
|
|
|
64
64
|
|
|
65
65
|
# File extensions to skip (from backend exclusion_patterns.py)
|
|
66
66
|
SKIP_EXTENSIONS = {
|
|
67
|
-
# Security - keys/certs
|
|
68
|
-
".pem", ".key", ".p12", ".pfx", ".crt", ".cer",
|
|
67
|
+
# Security - keys/certs/GPG
|
|
68
|
+
".pem", ".key", ".p12", ".pfx", ".crt", ".cer", ".asc",
|
|
69
69
|
# Python compiled
|
|
70
70
|
".pyc", ".pyo", ".pyd", ".egg",
|
|
71
71
|
# JVM
|
main.py
CHANGED
|
@@ -12,6 +12,7 @@ Usage:
|
|
|
12
12
|
nia link ID PATH # Link a source to local folder
|
|
13
13
|
"""
|
|
14
14
|
import os
|
|
15
|
+
import json
|
|
15
16
|
import typer
|
|
16
17
|
import httpx
|
|
17
18
|
import logging
|
|
@@ -171,15 +172,54 @@ def upgrade():
|
|
|
171
172
|
|
|
172
173
|
|
|
173
174
|
@app.command()
|
|
174
|
-
def status(
|
|
175
|
+
def status(
|
|
176
|
+
json_output: bool = typer.Option(False, "--json", "-j", help="Output as JSON for programmatic use"),
|
|
177
|
+
):
|
|
175
178
|
"""Show sync status and configured sources."""
|
|
176
179
|
if not is_authenticated():
|
|
177
|
-
|
|
180
|
+
if json_output:
|
|
181
|
+
print(json.dumps({"error": "Not logged in"}))
|
|
182
|
+
else:
|
|
183
|
+
console.print("[red]Not logged in. Run [cyan]nia login[/cyan] first.[/red]")
|
|
178
184
|
raise typer.Exit(1)
|
|
179
185
|
|
|
186
|
+
sources = get_sources()
|
|
187
|
+
|
|
188
|
+
if json_output:
|
|
189
|
+
output = []
|
|
190
|
+
for source in sources:
|
|
191
|
+
source_id = source.get("local_folder_id", "")
|
|
192
|
+
name = source.get("display_name", "")
|
|
193
|
+
path = source.get("path") or ""
|
|
194
|
+
detected_type = source.get("detected_type") or "folder"
|
|
195
|
+
|
|
196
|
+
# Determine status
|
|
197
|
+
if path:
|
|
198
|
+
expanded = os.path.expanduser(path)
|
|
199
|
+
status_str = "ready" if os.path.exists(expanded) else "path_not_found"
|
|
200
|
+
elif detected_type in KNOWN_PATHS:
|
|
201
|
+
known_path = os.path.expanduser(KNOWN_PATHS[detected_type])
|
|
202
|
+
if os.path.exists(known_path):
|
|
203
|
+
status_str = "ready"
|
|
204
|
+
path = KNOWN_PATHS[detected_type]
|
|
205
|
+
else:
|
|
206
|
+
status_str = "not_found_locally"
|
|
207
|
+
else:
|
|
208
|
+
status_str = "needs_link"
|
|
209
|
+
|
|
210
|
+
output.append({
|
|
211
|
+
"id": source_id,
|
|
212
|
+
"name": name,
|
|
213
|
+
"path": path,
|
|
214
|
+
"type": detected_type,
|
|
215
|
+
"status": status_str,
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
print(json.dumps({"sources": output}, indent=2))
|
|
219
|
+
return
|
|
220
|
+
|
|
180
221
|
console.print("[bold cyan]Nia Sync Status[/bold cyan]\n")
|
|
181
222
|
|
|
182
|
-
sources = get_sources()
|
|
183
223
|
if not sources:
|
|
184
224
|
console.print("[yellow]No sources configured.[/yellow]")
|
|
185
225
|
console.print("\n[dim]Add sources in the Nia web app, or run:[/dim] [cyan]nia add ~/path/to/folder[/cyan]")
|
|
@@ -203,23 +243,23 @@ def status():
|
|
|
203
243
|
if path:
|
|
204
244
|
expanded = os.path.expanduser(path)
|
|
205
245
|
if os.path.exists(expanded):
|
|
206
|
-
|
|
246
|
+
status_str = "[green]✓ ready[/green]"
|
|
207
247
|
else:
|
|
208
|
-
|
|
248
|
+
status_str = "[yellow]○ path not found[/yellow]"
|
|
209
249
|
else:
|
|
210
250
|
# Check if it's a known type we can auto-detect
|
|
211
251
|
if detected_type in KNOWN_PATHS:
|
|
212
252
|
known_path = os.path.expanduser(KNOWN_PATHS[detected_type])
|
|
213
253
|
if os.path.exists(known_path):
|
|
214
|
-
|
|
254
|
+
status_str = "[green]✓ ready[/green]"
|
|
215
255
|
path = KNOWN_PATHS[detected_type]
|
|
216
256
|
else:
|
|
217
|
-
|
|
257
|
+
status_str = "[yellow]○ not found locally[/yellow]"
|
|
218
258
|
else:
|
|
219
|
-
|
|
259
|
+
status_str = "[red]⚠ needs link[/red]"
|
|
220
260
|
needs_link.append(source)
|
|
221
261
|
|
|
222
|
-
table.add_row(source_id, name, path or "[dim]not set[/dim]", detected_type,
|
|
262
|
+
table.add_row(source_id, name, path or "[dim]not set[/dim]", detected_type, status_str)
|
|
223
263
|
|
|
224
264
|
console.print(table)
|
|
225
265
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
auth.py,sha256=n0ezRqIbz3kZYcyIjH47_MDKgwNgAaVr0Y2NEfRxa50,5704
|
|
2
|
+
config.py,sha256=dbnkDihOiJL0-WBzTyxDXM2AIduwsvZycgQuhAoYglU,7637
|
|
3
|
+
extractor.py,sha256=IxpFGOlklusmPs2Jz398DH1hVVYpciwQN_HDyxpmm_Q,29193
|
|
4
|
+
main.py,sha256=f1e2q99QfI_U0YyYgsC75P1wHebKuweWa0pYEAQPMO8,26582
|
|
5
|
+
sync.py,sha256=-oI7Os3LkrRiIHvLcPcmqsr4v4_aQHwf0T2DpeRu-n4,10406
|
|
6
|
+
watcher.py,sha256=JmsN9uR7Ss1mDC-kApXL6Hg_wQZWTsO7rRIFkQu8GbM,9978
|
|
7
|
+
nia_sync-0.1.4.dist-info/METADATA,sha256=2lMkL4_UpGoWEewNvEmJsOn9ktCMnZM5HOUWHplqKqg,246
|
|
8
|
+
nia_sync-0.1.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
9
|
+
nia_sync-0.1.4.dist-info/entry_points.txt,sha256=Fx8TIOgXqWdZzZEkEateDtcNfgnwuPW4jZTqlEUrHVs,33
|
|
10
|
+
nia_sync-0.1.4.dist-info/top_level.txt,sha256=_ZWBugSHWwSpLXYJAcF6TlWmzECu18k0y1-EX27jtBw,40
|
|
11
|
+
nia_sync-0.1.4.dist-info/RECORD,,
|
sync.py
CHANGED
|
@@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
|
|
|
22
22
|
SYNC_TIMEOUT = 60 # 1 minute per sync request (reduced from 2 min)
|
|
23
23
|
CONNECT_TIMEOUT = 10 # 10 second connection timeout
|
|
24
24
|
MAX_FILES_PER_BATCH = 500 # Keep below backend limit (1000)
|
|
25
|
+
MAX_BATCH_SIZE_BYTES = 4 * 1024 * 1024 # 4MB max payload per batch
|
|
25
26
|
MAX_RETRIES = 4
|
|
26
27
|
RETRY_BASE_DELAY = 1.5
|
|
27
28
|
RETRY_MAX_DELAY = 15.0
|
|
@@ -209,6 +210,8 @@ def upload_sync_data(
|
|
|
209
210
|
return {"status": "error", "message": "Authentication failed"}
|
|
210
211
|
elif response.status_code == 404:
|
|
211
212
|
return {"status": "error", "message": "Local folder not found"}
|
|
213
|
+
elif response.status_code == 413:
|
|
214
|
+
return {"status": "error", "message": "Request too large - please report this bug"}
|
|
212
215
|
else:
|
|
213
216
|
try:
|
|
214
217
|
detail = response.json().get("detail", response.text)
|
|
@@ -232,10 +235,11 @@ def upload_sync_batches(
|
|
|
232
235
|
if not files:
|
|
233
236
|
return {"status": "ok", "chunks_indexed": 0}
|
|
234
237
|
|
|
235
|
-
|
|
238
|
+
batches = list(_iter_batches_by_size(files, MAX_BATCH_SIZE_BYTES, MAX_FILES_PER_BATCH))
|
|
239
|
+
total_batches = len(batches)
|
|
236
240
|
chunks_indexed = 0
|
|
237
241
|
|
|
238
|
-
for batch_index, batch in enumerate(
|
|
242
|
+
for batch_index, batch in enumerate(batches, start=1):
|
|
239
243
|
is_last_batch = batch_index == total_batches
|
|
240
244
|
result = upload_sync_data(
|
|
241
245
|
local_folder_id=local_folder_id,
|
|
@@ -273,9 +277,33 @@ def report_sync_error(local_folder_id: str | None, error: str, path: str | None
|
|
|
273
277
|
logger.debug("Failed to report sync error", exc_info=True)
|
|
274
278
|
|
|
275
279
|
|
|
276
|
-
def
|
|
277
|
-
|
|
278
|
-
|
|
280
|
+
def _iter_batches_by_size(
|
|
281
|
+
files: list[dict[str, Any]],
|
|
282
|
+
max_bytes: int,
|
|
283
|
+
max_count: int,
|
|
284
|
+
):
|
|
285
|
+
"""Batch files by total payload size, not just count."""
|
|
286
|
+
batch: list[dict[str, Any]] = []
|
|
287
|
+
batch_size = 0
|
|
288
|
+
|
|
289
|
+
for f in files:
|
|
290
|
+
content = f.get("content", "")
|
|
291
|
+
file_size = len(content.encode("utf-8")) if content else 0
|
|
292
|
+
|
|
293
|
+
if file_size > max_bytes:
|
|
294
|
+
logger.warning(f"Skipping oversized file: {f.get('path')} ({file_size} bytes)")
|
|
295
|
+
continue
|
|
296
|
+
|
|
297
|
+
if batch and (batch_size + file_size > max_bytes or len(batch) >= max_count):
|
|
298
|
+
yield batch
|
|
299
|
+
batch = []
|
|
300
|
+
batch_size = 0
|
|
301
|
+
|
|
302
|
+
batch.append(f)
|
|
303
|
+
batch_size += file_size
|
|
304
|
+
|
|
305
|
+
if batch:
|
|
306
|
+
yield batch
|
|
279
307
|
|
|
280
308
|
|
|
281
309
|
def _post_with_retries(
|
nia_sync-0.1.2.dist-info/RECORD
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
auth.py,sha256=n0ezRqIbz3kZYcyIjH47_MDKgwNgAaVr0Y2NEfRxa50,5704
|
|
2
|
-
config.py,sha256=JWxdL8INKo23lam4F49ZbllxbW3yorqczm7aaqepQCc,7507
|
|
3
|
-
extractor.py,sha256=Y5Gc0AThTH9qMey4fYEqv4RosAqZl7GleKNMYs5ogxY,29181
|
|
4
|
-
main.py,sha256=NtAkI083bZIXZp9WVGaCunveV8UrR0ZBctBkBhrVf24,25168
|
|
5
|
-
sync.py,sha256=OrbjptK223zbzeelSTdZIF3O4gNQd5DLK8XXv-rnMXU,9583
|
|
6
|
-
watcher.py,sha256=JmsN9uR7Ss1mDC-kApXL6Hg_wQZWTsO7rRIFkQu8GbM,9978
|
|
7
|
-
nia_sync-0.1.2.dist-info/METADATA,sha256=Ihc9yJ1P_J0e28skRXVUgnh6PXdltk59Zcijw9YZrt8,246
|
|
8
|
-
nia_sync-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
9
|
-
nia_sync-0.1.2.dist-info/entry_points.txt,sha256=Fx8TIOgXqWdZzZEkEateDtcNfgnwuPW4jZTqlEUrHVs,33
|
|
10
|
-
nia_sync-0.1.2.dist-info/top_level.txt,sha256=_ZWBugSHWwSpLXYJAcF6TlWmzECu18k0y1-EX27jtBw,40
|
|
11
|
-
nia_sync-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|