copyparty 1.15.7__py3-none-any.whl → 1.15.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
copyparty/__main__.py CHANGED
@@ -1009,7 +1009,7 @@ def add_upload(ap):
1009
1009
  ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
1010
1010
  ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m-1\033[0m] = forbidden/always-off, [\033[32m0\033[0m] = default-off and warn if enabled, [\033[32m1\033[0m] = default-off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
1011
1011
  ap2.add_argument("--u2j", metavar="JOBS", type=int, default=2, help="web-client: number of file chunks to upload in parallel; 1 or 2 is good for low-latency (same-country) connections, 4-8 for android clients, 16 for cross-atlantic (max=64)")
1012
- ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for this size. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
1012
+ ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for \033[33mdefault\033[0m, and never exceed \033[33mmax\033[0m. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
1013
1013
  ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
1014
1014
  ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
1015
1015
 
@@ -1029,7 +1029,7 @@ def add_network(ap):
1029
1029
  else:
1030
1030
  ap2.add_argument("--freebind", action="store_true", help="allow listening on IPs which do not yet exist, for example if the network interfaces haven't finished going up. Only makes sense for IPs other than '0.0.0.0', '127.0.0.1', '::', and '::1'. May require running as root (unless net.ipv6.ip_nonlocal_bind)")
1031
1031
  ap2.add_argument("--s-thead", metavar="SEC", type=int, default=120, help="socket timeout (read request header)")
1032
- ap2.add_argument("--s-tbody", metavar="SEC", type=float, default=186.0, help="socket timeout (read/write request/response bodies). Use 60 on fast servers (default is extremely safe). Disable with 0 if reverse-proxied for a 2%% speed boost")
1032
+ ap2.add_argument("--s-tbody", metavar="SEC", type=float, default=128.0, help="socket timeout (read/write request/response bodies). Use 60 on fast servers (default is extremely safe). Disable with 0 if reverse-proxied for a 2%% speed boost")
1033
1033
  ap2.add_argument("--s-rd-sz", metavar="B", type=int, default=256*1024, help="socket read size in bytes (indirectly affects filesystem writes; recommendation: keep equal-to or lower-than \033[33m--iobuf\033[0m)")
1034
1034
  ap2.add_argument("--s-wr-sz", metavar="B", type=int, default=256*1024, help="socket write size in bytes")
1035
1035
  ap2.add_argument("--s-wr-slp", metavar="SEC", type=float, default=0.0, help="debug: socket write delay in seconds")
@@ -1349,6 +1349,14 @@ def add_transcoding(ap):
1349
1349
  ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete cached transcode output after \033[33mSEC\033[0m seconds")
1350
1350
 
1351
1351
 
1352
+ def add_rss(ap):
1353
+ ap2 = ap.add_argument_group('RSS options')
1354
+ ap2.add_argument("--rss", action="store_true", help="enable RSS output (experimental)")
1355
+ ap2.add_argument("--rss-nf", metavar="HITS", type=int, default=250, help="default number of files to return (url-param 'nf')")
1356
+ ap2.add_argument("--rss-fext", metavar="E,E", type=u, default="", help="default list of file extensions to include (url-param 'fext'); blank=all")
1357
+ ap2.add_argument("--rss-sort", metavar="ORD", type=u, default="m", help="default sort order (url-param 'sort'); [\033[32mm\033[0m]=last-modified [\033[32mu\033[0m]=upload-time [\033[32mn\033[0m]=filename [\033[32ms\033[0m]=filesize; Uppercase=oldest-first. Note that upload-time is 0 for non-uploaded files")
1358
+
1359
+
1352
1360
  def add_db_general(ap, hcores):
1353
1361
  noidx = APPLESAN_TXT if MACOS else ""
1354
1362
  ap2 = ap.add_argument_group('general db options')
@@ -1518,6 +1526,7 @@ def run_argparse(
1518
1526
  add_db_metadata(ap)
1519
1527
  add_thumbnail(ap)
1520
1528
  add_transcoding(ap)
1529
+ add_rss(ap)
1521
1530
  add_ftp(ap)
1522
1531
  add_webdav(ap)
1523
1532
  add_tftp(ap)
copyparty/__version__.py CHANGED
@@ -1,8 +1,8 @@
1
1
  # coding: utf-8
2
2
 
3
- VERSION = (1, 15, 7)
3
+ VERSION = (1, 15, 9)
4
4
  CODENAME = "fill the drives"
5
- BUILD_DT = (2024, 10, 14)
5
+ BUILD_DT = (2024, 10, 18)
6
6
 
7
7
  S_VERSION = ".".join(map(str, VERSION))
8
8
  S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
copyparty/authsrv.py CHANGED
@@ -59,6 +59,7 @@ if PY2:
59
59
  LEELOO_DALLAS = "leeloo_dallas"
60
60
 
61
61
  SEE_LOG = "see log for details"
62
+ SEESLOG = " (see serverlog for details)"
62
63
  SSEELOG = " ({})".format(SEE_LOG)
63
64
  BAD_CFG = "invalid config; {}".format(SEE_LOG)
64
65
  SBADCFG = " ({})".format(BAD_CFG)
@@ -157,8 +158,11 @@ class Lim(object):
157
158
  self.chk_rem(rem)
158
159
  if sz != -1:
159
160
  self.chk_sz(sz)
160
- self.chk_vsz(broker, ptop, sz, volgetter)
161
- self.chk_df(abspath, sz) # side effects; keep last-ish
161
+ else:
162
+ sz = 0
163
+
164
+ self.chk_vsz(broker, ptop, sz, volgetter)
165
+ self.chk_df(abspath, sz) # side effects; keep last-ish
162
166
 
163
167
  ap2, vp2 = self.rot(abspath)
164
168
  if abspath == ap2:
@@ -198,7 +202,15 @@ class Lim(object):
198
202
 
199
203
  if self.dft < time.time():
200
204
  self.dft = int(time.time()) + 300
201
- self.dfv = get_df(abspath)[0] or 0
205
+
206
+ df, du, err = get_df(abspath, True)
207
+ if err:
208
+ t = "failed to read disk space usage for [%s]: %s"
209
+ self.log(t % (abspath, err), 3)
210
+ self.dfv = 0xAAAAAAAAA # 42.6 GiB
211
+ else:
212
+ self.dfv = df or 0
213
+
202
214
  for j in list(self.reg.values()) if self.reg else []:
203
215
  self.dfv -= int(j["size"] / (len(j["hash"]) or 999) * len(j["need"]))
204
216
 
copyparty/cfg.py CHANGED
@@ -46,6 +46,7 @@ def vf_bmap() :
46
46
  "og_no_head",
47
47
  "og_s_title",
48
48
  "rand",
49
+ "rss",
49
50
  "xdev",
50
51
  "xlink",
51
52
  "xvol",
copyparty/httpcli.py CHANGED
@@ -127,6 +127,8 @@ LOGUES = [[0, ".prologue.html"], [1, ".epilogue.html"]]
127
127
 
128
128
  READMES = [[0, ["preadme.md", "PREADME.md"]], [1, ["readme.md", "README.md"]]]
129
129
 
130
+ RSS_SORT = {"m": "mt", "u": "at", "n": "fn", "s": "sz"}
131
+
130
132
 
131
133
  class HttpCli(object):
132
134
  """
@@ -1196,8 +1198,146 @@ class HttpCli(object):
1196
1198
  if "h" in self.uparam:
1197
1199
  return self.tx_mounts()
1198
1200
 
1201
+ if "rss" in self.uparam:
1202
+ return self.tx_rss()
1203
+
1199
1204
  return self.tx_browser()
1200
1205
 
1206
+ def tx_rss(self) :
1207
+ if self.do_log:
1208
+ self.log("RSS %s @%s" % (self.req, self.uname))
1209
+
1210
+ if not self.can_read:
1211
+ return self.tx_404()
1212
+
1213
+ vn = self.vn
1214
+ if not vn.flags.get("rss"):
1215
+ raise Pebkac(405, "RSS is disabled in server config")
1216
+
1217
+ rem = self.rem
1218
+ idx = self.conn.get_u2idx()
1219
+ if not idx or not hasattr(idx, "p_end"):
1220
+ if not HAVE_SQLITE3:
1221
+ raise Pebkac(500, "sqlite3 not found on server; rss is disabled")
1222
+ raise Pebkac(500, "server busy, cannot generate rss; please retry in a bit")
1223
+
1224
+ uv = [rem]
1225
+ if "recursive" in self.uparam:
1226
+ uq = "up.rd like ?||'%'"
1227
+ else:
1228
+ uq = "up.rd == ?"
1229
+
1230
+ zs = str(self.uparam.get("fext", self.args.rss_fext))
1231
+ if zs in ("True", "False"):
1232
+ zs = ""
1233
+ if zs:
1234
+ zsl = []
1235
+ for ext in zs.split(","):
1236
+ zsl.append("+up.fn like '%.'||?")
1237
+ uv.append(ext)
1238
+ uq += " and ( %s )" % (" or ".join(zsl),)
1239
+
1240
+ zs1 = self.uparam.get("sort", self.args.rss_sort)
1241
+ zs2 = zs1.lower()
1242
+ zs = RSS_SORT.get(zs2)
1243
+ if not zs:
1244
+ raise Pebkac(400, "invalid sort key; must be m/u/n/s")
1245
+
1246
+ uq += " order by up." + zs
1247
+ if zs1 == zs2:
1248
+ uq += " desc"
1249
+
1250
+ nmax = int(self.uparam.get("nf") or self.args.rss_nf)
1251
+
1252
+ hits = idx.run_query(self.uname, [self.vn], uq, uv, False, False, nmax)[0]
1253
+
1254
+ pw = self.ouparam.get("pw")
1255
+ if pw:
1256
+ q_pw = "?pw=%s" % (pw,)
1257
+ a_pw = "&pw=%s" % (pw,)
1258
+ for i in hits:
1259
+ i["rp"] += a_pw if "?" in i["rp"] else q_pw
1260
+ else:
1261
+ q_pw = a_pw = ""
1262
+
1263
+ title = self.uparam.get("title") or self.vpath.split("/")[-1]
1264
+ etitle = html_escape(title, True, True)
1265
+
1266
+ baseurl = "%s://%s%s" % (
1267
+ "https" if self.is_https else "http",
1268
+ self.host,
1269
+ self.args.SRS,
1270
+ )
1271
+ feed = "%s%s" % (baseurl, self.req[1:])
1272
+ efeed = html_escape(feed, True, True)
1273
+ edirlink = efeed.split("?")[0] + q_pw
1274
+
1275
+ ret = [
1276
+ """\
1277
+ <?xml version="1.0" encoding="UTF-8"?>
1278
+ <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:content="http://purl.org/rss/1.0/modules/content/">
1279
+ \t<channel>
1280
+ \t\t<atom:link href="%s" rel="self" type="application/rss+xml" />
1281
+ \t\t<title>%s</title>
1282
+ \t\t<description></description>
1283
+ \t\t<link>%s</link>
1284
+ \t\t<generator>copyparty-1</generator>
1285
+ """
1286
+ % (efeed, etitle, edirlink)
1287
+ ]
1288
+
1289
+ q = "select fn from cv where rd=? and dn=?"
1290
+ crd, cdn = rem.rsplit("/", 1) if "/" in rem else ("", rem)
1291
+ try:
1292
+ cfn = idx.cur[self.vn.realpath].execute(q, (crd, cdn)).fetchone()[0]
1293
+ bos.stat(os.path.join(vn.canonical(rem), cfn))
1294
+ cv_url = "%s%s?th=jf%s" % (baseurl, vjoin(self.vpath, cfn), a_pw)
1295
+ cv_url = html_escape(cv_url, True, True)
1296
+ zs = """\
1297
+ \t\t<image>
1298
+ \t\t\t<url>%s</url>
1299
+ \t\t\t<title>%s</title>
1300
+ \t\t\t<link>%s</link>
1301
+ \t\t</image>
1302
+ """
1303
+ ret.append(zs % (cv_url, etitle, edirlink))
1304
+ except:
1305
+ pass
1306
+
1307
+ for i in hits:
1308
+ iurl = html_escape("%s%s" % (baseurl, i["rp"]), True, True)
1309
+ title = unquotep(i["rp"].split("?")[0].split("/")[-1])
1310
+ title = html_escape(title, True, True)
1311
+ tag_t = str(i["tags"].get("title") or "")
1312
+ tag_a = str(i["tags"].get("artist") or "")
1313
+ desc = "%s - %s" % (tag_a, tag_t) if tag_t and tag_a else (tag_t or tag_a)
1314
+ desc = html_escape(desc, True, True) if desc else title
1315
+ mime = html_escape(guess_mime(title))
1316
+ lmod = formatdate(i["ts"])
1317
+ zsa = (iurl, iurl, title, desc, lmod, iurl, mime, i["sz"])
1318
+ zs = (
1319
+ """\
1320
+ \t\t<item>
1321
+ \t\t\t<guid>%s</guid>
1322
+ \t\t\t<link>%s</link>
1323
+ \t\t\t<title>%s</title>
1324
+ \t\t\t<description>%s</description>
1325
+ \t\t\t<pubDate>%s</pubDate>
1326
+ \t\t\t<enclosure url="%s" type="%s" length="%d"/>
1327
+ """
1328
+ % zsa
1329
+ )
1330
+ dur = i["tags"].get(".dur")
1331
+ if dur:
1332
+ zs += "\t\t\t<itunes:duration>%d</itunes:duration>\n" % (dur,)
1333
+ ret.append(zs + "\t\t</item>\n")
1334
+
1335
+ ret.append("\t</channel>\n</rss>\n")
1336
+ bret = "".join(ret).encode("utf-8", "replace")
1337
+ self.reply(bret, 200, "text/xml; charset=utf-8")
1338
+ self.log("rss: %d hits, %d bytes" % (len(hits), len(bret)))
1339
+ return True
1340
+
1201
1341
  def handle_propfind(self) :
1202
1342
  if self.do_log:
1203
1343
  self.log("PFIND %s @%s" % (self.req, self.uname))
@@ -1874,7 +2014,7 @@ class HttpCli(object):
1874
2014
  f, fn = ren_open(fn, *open_a, **params)
1875
2015
  try:
1876
2016
  path = os.path.join(fdir, fn)
1877
- post_sz, sha_hex, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
2017
+ post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp)
1878
2018
  finally:
1879
2019
  f.close()
1880
2020
 
@@ -2337,7 +2477,7 @@ class HttpCli(object):
2337
2477
  broker = self.conn.hsrv.broker
2338
2478
  x = broker.ask("up2k.handle_chunks", ptop, wark, chashes)
2339
2479
  response = x.get()
2340
- chashes, chunksize, cstarts, path, lastmod, sprs = response
2480
+ chashes, chunksize, cstarts, path, lastmod, fsize, sprs = response
2341
2481
  maxsize = chunksize * len(chashes)
2342
2482
  cstart0 = cstarts[0]
2343
2483
  locked = chashes # remaining chunks to be received in this request
@@ -2345,6 +2485,50 @@ class HttpCli(object):
2345
2485
  num_left = -1 # num chunks left according to most recent up2k release
2346
2486
  treport = time.time() # ratelimit up2k reporting to reduce overhead
2347
2487
 
2488
+ if "x-up2k-subc" in self.headers:
2489
+ sc_ofs = int(self.headers["x-up2k-subc"])
2490
+ chash = chashes[0]
2491
+
2492
+ u2sc = self.conn.hsrv.u2sc
2493
+ try:
2494
+ sc_pofs, hasher = u2sc[chash]
2495
+ if not sc_ofs:
2496
+ t = "client restarted the chunk; forgetting subchunk offset %d"
2497
+ self.log(t % (sc_pofs,))
2498
+ raise Exception()
2499
+ except:
2500
+ sc_pofs = 0
2501
+ hasher = hashlib.sha512()
2502
+
2503
+ et = "subchunk protocol error; resetting chunk "
2504
+ if sc_pofs != sc_ofs:
2505
+ u2sc.pop(chash, None)
2506
+ t = "%s[%s]: the expected resume-point was %d, not %d"
2507
+ raise Pebkac(400, t % (et, chash, sc_pofs, sc_ofs))
2508
+ if len(cstarts) > 1:
2509
+ u2sc.pop(chash, None)
2510
+ t = "%s[%s]: only a single subchunk can be uploaded in one request; you are sending %d chunks"
2511
+ raise Pebkac(400, t % (et, chash, len(cstarts)))
2512
+ csize = min(chunksize, fsize - cstart0[0])
2513
+ cstart0[0] += sc_ofs # also sets cstarts[0][0]
2514
+ sc_next_ofs = sc_ofs + postsize
2515
+ if sc_next_ofs > csize:
2516
+ u2sc.pop(chash, None)
2517
+ t = "%s[%s]: subchunk offset (%d) plus postsize (%d) exceeds chunksize (%d)"
2518
+ raise Pebkac(400, t % (et, chash, sc_ofs, postsize, csize))
2519
+ else:
2520
+ final_subchunk = sc_next_ofs == csize
2521
+ t = "subchunk %s %d:%d/%d %s"
2522
+ zs = "END" if final_subchunk else ""
2523
+ self.log(t % (chash[:15], sc_ofs, sc_next_ofs, csize, zs), 6)
2524
+ if final_subchunk:
2525
+ u2sc.pop(chash, None)
2526
+ else:
2527
+ u2sc[chash] = (sc_next_ofs, hasher)
2528
+ else:
2529
+ hasher = None
2530
+ final_subchunk = True
2531
+
2348
2532
  try:
2349
2533
  if self.args.nw:
2350
2534
  path = os.devnull
@@ -2375,9 +2559,11 @@ class HttpCli(object):
2375
2559
  reader = read_socket(
2376
2560
  self.sr, self.args.s_rd_sz, min(remains, chunksize)
2377
2561
  )
2378
- post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
2562
+ post_sz, _, sha_b64 = hashcopy(
2563
+ reader, f, hasher, 0, self.args.s_wr_slp
2564
+ )
2379
2565
 
2380
- if sha_b64 != chash:
2566
+ if sha_b64 != chash and final_subchunk:
2381
2567
  try:
2382
2568
  self.bakflip(
2383
2569
  f, path, cstart[0], post_sz, chash, sha_b64, vfs.flags
@@ -2409,7 +2595,8 @@ class HttpCli(object):
2409
2595
 
2410
2596
  # be quick to keep the tcp winsize scale;
2411
2597
  # if we can't confirm rn then that's fine
2412
- written.append(chash)
2598
+ if final_subchunk:
2599
+ written.append(chash)
2413
2600
  now = time.time()
2414
2601
  if now - treport < 1:
2415
2602
  continue
@@ -2434,6 +2621,7 @@ class HttpCli(object):
2434
2621
  except:
2435
2622
  # maybe busted handle (eg. disk went full)
2436
2623
  f.close()
2624
+ chashes = [] # exception flag
2437
2625
  raise
2438
2626
  finally:
2439
2627
  if locked:
@@ -2442,9 +2630,11 @@ class HttpCli(object):
2442
2630
  num_left, t = x.get()
2443
2631
  if num_left < 0:
2444
2632
  self.loud_reply(t, status=500)
2445
- return False
2446
- t = "got %d more chunks, %d left"
2447
- self.log(t % (len(written), num_left), 6)
2633
+ if chashes: # kills exception bubbling otherwise
2634
+ return False
2635
+ else:
2636
+ t = "got %d more chunks, %d left"
2637
+ self.log(t % (len(written), num_left), 6)
2448
2638
 
2449
2639
  if num_left < 0:
2450
2640
  raise Pebkac(500, "unconfirmed; see serverlog")
@@ -2797,7 +2987,7 @@ class HttpCli(object):
2797
2987
  tabspath = os.path.join(fdir, tnam)
2798
2988
  self.log("writing to {}".format(tabspath))
2799
2989
  sz, sha_hex, sha_b64 = hashcopy(
2800
- p_data, f, self.args.s_wr_slp, max_sz
2990
+ p_data, f, None, max_sz, self.args.s_wr_slp
2801
2991
  )
2802
2992
  if sz == 0:
2803
2993
  raise Pebkac(400, "empty files in post")
@@ -3127,7 +3317,7 @@ class HttpCli(object):
3127
3317
  wunlink(self.log, fp, vfs.flags)
3128
3318
 
3129
3319
  with open(fsenc(fp), "wb", self.args.iobuf) as f:
3130
- sz, sha512, _ = hashcopy(p_data, f, self.args.s_wr_slp)
3320
+ sz, sha512, _ = hashcopy(p_data, f, None, 0, self.args.s_wr_slp)
3131
3321
 
3132
3322
  if lim:
3133
3323
  lim.nup(self.ip)
@@ -5035,7 +5225,7 @@ class HttpCli(object):
5035
5225
  self.log("#wow #whoa")
5036
5226
 
5037
5227
  if not self.args.nid:
5038
- free, total = get_df(abspath)
5228
+ free, total, _ = get_df(abspath, False)
5039
5229
  if total is not None:
5040
5230
  h1 = humansize(free or 0)
5041
5231
  h2 = humansize(total)
copyparty/httpsrv.py CHANGED
@@ -1,6 +1,7 @@
1
1
  # coding: utf-8
2
2
  from __future__ import print_function, unicode_literals
3
3
 
4
+ import hashlib
4
5
  import math
5
6
  import os
6
7
  import re
@@ -141,6 +142,7 @@ class HttpSrv(object):
141
142
  self.t_periodic = None
142
143
 
143
144
  self.u2fh = FHC()
145
+ self.u2sc = {}
144
146
  self.pipes = CachedDict(0.2)
145
147
  self.metrics = Metrics(self)
146
148
  self.nreq = 0
copyparty/metrics.py CHANGED
@@ -128,7 +128,7 @@ class Metrics(object):
128
128
  addbh("cpp_disk_size_bytes", "total HDD size of volume")
129
129
  addbh("cpp_disk_free_bytes", "free HDD space in volume")
130
130
  for vpath, vol in allvols:
131
- free, total = get_df(vol.realpath)
131
+ free, total, _ = get_df(vol.realpath, False)
132
132
  if free is None or total is None:
133
133
  continue
134
134
 
copyparty/u2idx.py CHANGED
@@ -91,7 +91,7 @@ class U2idx(object):
91
91
  uv = [wark[:16], wark]
92
92
 
93
93
  try:
94
- return self.run_query(uname, vols, uq, uv, False, 99999)[0]
94
+ return self.run_query(uname, vols, uq, uv, False, True, 99999)[0]
95
95
  except:
96
96
  raise Pebkac(500, min_ex())
97
97
 
@@ -295,7 +295,7 @@ class U2idx(object):
295
295
  q += " lower({}) {} ? ) ".format(field, oper)
296
296
 
297
297
  try:
298
- return self.run_query(uname, vols, q, va, have_mt, lim)
298
+ return self.run_query(uname, vols, q, va, have_mt, True, lim)
299
299
  except Exception as ex:
300
300
  raise Pebkac(500, repr(ex))
301
301
 
@@ -306,6 +306,7 @@ class U2idx(object):
306
306
  uq ,
307
307
  uv ,
308
308
  have_mt ,
309
+ sort ,
309
310
  lim ,
310
311
  ) :
311
312
  if self.args.srch_dbg:
@@ -452,7 +453,8 @@ class U2idx(object):
452
453
  done_flag.append(True)
453
454
  self.active_id = ""
454
455
 
455
- ret.sort(key=itemgetter("rp"))
456
+ if sort:
457
+ ret.sort(key=itemgetter("rp"))
456
458
 
457
459
  return ret, list(taglist.keys()), lim < 0 and not clamped
458
460
 
copyparty/up2k.py CHANGED
@@ -20,7 +20,7 @@ from copy import deepcopy
20
20
  from queue import Queue
21
21
 
22
22
  from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E
23
- from .authsrv import LEELOO_DALLAS, SSEELOG, VFS, AuthSrv
23
+ from .authsrv import LEELOO_DALLAS, SEESLOG, VFS, AuthSrv
24
24
  from .bos import bos
25
25
  from .cfg import vf_bmap, vf_cmap, vf_vmap
26
26
  from .fsutil import Fstab
@@ -2876,9 +2876,6 @@ class Up2k(object):
2876
2876
  "user": cj["user"],
2877
2877
  "addr": ip,
2878
2878
  "at": at,
2879
- "hash": [],
2880
- "need": [],
2881
- "busy": {},
2882
2879
  }
2883
2880
  for k in ["life"]:
2884
2881
  if k in cj:
@@ -2912,17 +2909,20 @@ class Up2k(object):
2912
2909
  hashes2, st = self._hashlist_from_file(orig_ap)
2913
2910
  wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
2914
2911
  if dwark != wark2:
2915
- t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s"
2916
- self.log(t % (wark2, dwark, orig_ap))
2912
+ t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s\n%s"
2913
+ self.log(t % (wark2, dwark, orig_ap, rj))
2917
2914
  lost.append(dupe[3:])
2918
2915
  continue
2919
2916
  data_ok = True
2920
2917
  job = rj
2921
2918
  break
2922
2919
 
2923
- if job and wark in reg:
2924
- # self.log("pop " + wark + " " + job["name"] + " handle_json db", 4)
2925
- del reg[wark]
2920
+ if job:
2921
+ if wark in reg:
2922
+ del reg[wark]
2923
+ job["hash"] = job["need"] = []
2924
+ job["done"] = True
2925
+ job["busy"] = {}
2926
2926
 
2927
2927
  if lost:
2928
2928
  c2 = None
@@ -2950,7 +2950,7 @@ class Up2k(object):
2950
2950
  path = djoin(rj["ptop"], rj["prel"], fn)
2951
2951
  try:
2952
2952
  st = bos.stat(path)
2953
- if st.st_size > 0 or not rj["need"]:
2953
+ if st.st_size > 0 or "done" in rj:
2954
2954
  # upload completed or both present
2955
2955
  break
2956
2956
  except:
@@ -2964,13 +2964,13 @@ class Up2k(object):
2964
2964
  inc_ap = djoin(cj["ptop"], cj["prel"], cj["name"])
2965
2965
  orig_ap = djoin(rj["ptop"], rj["prel"], rj["name"])
2966
2966
 
2967
- if self.args.nw or n4g or not st:
2967
+ if self.args.nw or n4g or not st or "done" not in rj:
2968
2968
  pass
2969
2969
 
2970
2970
  elif st.st_size != rj["size"]:
2971
- t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}"
2971
+ t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}\n{}"
2972
2972
  t = t.format(
2973
- wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path
2973
+ wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path, rj
2974
2974
  )
2975
2975
  self.log(t)
2976
2976
  del reg[wark]
@@ -2980,8 +2980,8 @@ class Up2k(object):
2980
2980
  hashes2, _ = self._hashlist_from_file(orig_ap)
2981
2981
  wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
2982
2982
  if wark != wark2:
2983
- t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s"
2984
- self.log(t % (wark2, wark, orig_ap))
2983
+ t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s\n%s"
2984
+ self.log(t % (wark2, wark, orig_ap, rj))
2985
2985
  del reg[wark]
2986
2986
 
2987
2987
  if job or wark in reg:
@@ -2996,7 +2996,7 @@ class Up2k(object):
2996
2996
  dst = djoin(cj["ptop"], cj["prel"], cj["name"])
2997
2997
  vsrc = djoin(job["vtop"], job["prel"], job["name"])
2998
2998
  vsrc = vsrc.replace("\\", "/") # just for prints anyways
2999
- if job["need"]:
2999
+ if "done" not in job:
3000
3000
  self.log("unfinished:\n {0}\n {1}".format(src, dst))
3001
3001
  err = "partial upload exists at a different location; please resume uploading here instead:\n"
3002
3002
  err += "/" + quotep(vsrc) + " "
@@ -3357,14 +3357,14 @@ class Up2k(object):
3357
3357
 
3358
3358
  def handle_chunks(
3359
3359
  self, ptop , wark , chashes
3360
- ) :
3360
+ ) :
3361
3361
  with self.mutex, self.reg_mutex:
3362
3362
  self.db_act = self.vol_act[ptop] = time.time()
3363
3363
  job = self.registry[ptop].get(wark)
3364
3364
  if not job:
3365
3365
  known = " ".join([x for x in self.registry[ptop].keys()])
3366
3366
  self.log("unknown wark [{}], known: {}".format(wark, known))
3367
- raise Pebkac(400, "unknown wark" + SSEELOG)
3367
+ raise Pebkac(400, "unknown wark" + SEESLOG)
3368
3368
 
3369
3369
  if "t0c" not in job:
3370
3370
  job["t0c"] = time.time()
@@ -3380,7 +3380,7 @@ class Up2k(object):
3380
3380
  try:
3381
3381
  nchunk = uniq.index(chashes[0])
3382
3382
  except:
3383
- raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0]))
3383
+ raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0],))
3384
3384
  expanded = [chashes[0]]
3385
3385
  for prefix in chashes[1:]:
3386
3386
  nchunk += 1
@@ -3414,7 +3414,7 @@ class Up2k(object):
3414
3414
  for chash in chashes:
3415
3415
  nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
3416
3416
  if not nchunk:
3417
- raise Pebkac(400, "unknown chunk %s" % (chash))
3417
+ raise Pebkac(400, "unknown chunk %s" % (chash,))
3418
3418
 
3419
3419
  ofs = [chunksize * x for x in nchunk]
3420
3420
  coffsets.append(ofs)
@@ -3439,7 +3439,7 @@ class Up2k(object):
3439
3439
 
3440
3440
  job["poke"] = time.time()
3441
3441
 
3442
- return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
3442
+ return chashes, chunksize, coffsets, path, job["lmod"], job["size"], job["sprs"]
3443
3443
 
3444
3444
  def fast_confirm_chunks(
3445
3445
  self, ptop , wark , chashes
@@ -3481,6 +3481,7 @@ class Up2k(object):
3481
3481
  for chash in written:
3482
3482
  job["need"].remove(chash)
3483
3483
  except Exception as ex:
3484
+ # dead tcp connections can get here by timeout (OK)
3484
3485
  return -2, "confirm_chunk, chash(%s) %r" % (chash, ex) # type: ignore
3485
3486
 
3486
3487
  ret = len(job["need"])
@@ -3508,11 +3509,13 @@ class Up2k(object):
3508
3509
  src = djoin(pdir, job["tnam"])
3509
3510
  dst = djoin(pdir, job["name"])
3510
3511
  except Exception as ex:
3511
- raise Pebkac(500, "finish_upload, wark, " + repr(ex))
3512
+ self.log(min_ex(), 1)
3513
+ raise Pebkac(500, "finish_upload, wark, %r%s" % (ex, SEESLOG))
3512
3514
 
3513
3515
  if job["need"]:
3514
- t = "finish_upload {} with remaining chunks {}"
3515
- raise Pebkac(500, t.format(wark, job["need"]))
3516
+ self.log(min_ex(), 1)
3517
+ t = "finish_upload %s with remaining chunks %s%s"
3518
+ raise Pebkac(500, t % (wark, job["need"], SEESLOG))
3516
3519
 
3517
3520
  upt = job.get("at") or time.time()
3518
3521
  vflags = self.flags[ptop]
@@ -4033,7 +4036,9 @@ class Up2k(object):
4033
4036
  self.db_act = self.vol_act[dbv.realpath] = time.time()
4034
4037
  svpf = "/".join(x for x in [dbv.vpath, vrem, fn[0]] if x)
4035
4038
  if not svpf.startswith(svp + "/"): # assert
4036
- raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
4039
+ self.log(min_ex(), 1)
4040
+ t = "mv: bug at %s, top %s%s"
4041
+ raise Pebkac(500, t % (svpf, svp, SEESLOG))
4037
4042
 
4038
4043
  dvpf = dvp + svpf[len(svp) :]
4039
4044
  self._mv_file(uname, ip, svpf, dvpf, curs)
@@ -4048,7 +4053,9 @@ class Up2k(object):
4048
4053
  for zsl in (rm_ok, rm_ng):
4049
4054
  for ap in reversed(zsl):
4050
4055
  if not ap.startswith(sabs):
4051
- raise Pebkac(500, "mv_d: bug at {}, top {}".format(ap, sabs))
4056
+ self.log(min_ex(), 1)
4057
+ t = "mv_d: bug at %s, top %s%s"
4058
+ raise Pebkac(500, t % (ap, sabs, SEESLOG))
4052
4059
 
4053
4060
  rem = ap[len(sabs) :].replace(os.sep, "/").lstrip("/")
4054
4061
  vp = vjoin(dvp, rem)
copyparty/util.py CHANGED
@@ -192,6 +192,9 @@ except:
192
192
  ansi_re = re.compile("\033\\[[^mK]*[mK]")
193
193
 
194
194
 
195
+ BOS_SEP = ("%s" % (os.sep,)).encode("ascii")
196
+
197
+
195
198
  surrogateescape.register_surrogateescape()
196
199
  if WINDOWS and PY2:
197
200
  FS_ENCODING = "utf-8"
@@ -2407,22 +2410,27 @@ def wunlink(log , abspath , flags ) :
2407
2410
  return _fs_mvrm(log, abspath, "", False, flags)
2408
2411
 
2409
2412
 
2410
- def get_df(abspath ) :
2413
+ def get_df(abspath , prune ) :
2411
2414
  try:
2412
- # some fuses misbehave
2415
+ ap = fsenc(abspath)
2416
+ while prune and not os.path.isdir(ap) and BOS_SEP in ap:
2417
+ # strip leafs until it hits an existing folder
2418
+ ap = ap.rsplit(BOS_SEP, 1)[0]
2419
+
2413
2420
  if ANYWIN:
2421
+ abspath = fsdec(ap)
2414
2422
  bfree = ctypes.c_ulonglong(0)
2415
2423
  ctypes.windll.kernel32.GetDiskFreeSpaceExW( # type: ignore
2416
2424
  ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
2417
2425
  )
2418
- return (bfree.value, None)
2426
+ return (bfree.value, None, "")
2419
2427
  else:
2420
- sv = os.statvfs(fsenc(abspath))
2428
+ sv = os.statvfs(ap)
2421
2429
  free = sv.f_frsize * sv.f_bfree
2422
2430
  total = sv.f_frsize * sv.f_blocks
2423
- return (free, total)
2424
- except:
2425
- return (None, None)
2431
+ return (free, total, "")
2432
+ except Exception as ex:
2433
+ return (None, None, repr(ex))
2426
2434
 
2427
2435
 
2428
2436
  if not ANYWIN and not MACOS:
@@ -2637,10 +2645,12 @@ def yieldfile(fn , bufsz ) :
2637
2645
  def hashcopy(
2638
2646
  fin ,
2639
2647
  fout ,
2640
- slp = 0,
2641
- max_sz = 0,
2648
+ hashobj ,
2649
+ max_sz ,
2650
+ slp ,
2642
2651
  ) :
2643
- hashobj = hashlib.sha512()
2652
+ if not hashobj:
2653
+ hashobj = hashlib.sha512()
2644
2654
  tlen = 0
2645
2655
  for buf in fin:
2646
2656
  tlen += len(buf)
copyparty/web/a/u2c.py CHANGED
@@ -1,8 +1,8 @@
1
1
  #!/usr/bin/env python3
2
2
  from __future__ import print_function, unicode_literals
3
3
 
4
- S_VERSION = "2.2"
5
- S_BUILD_DT = "2024-10-13"
4
+ S_VERSION = "2.5"
5
+ S_BUILD_DT = "2024-10-18"
6
6
 
7
7
  """
8
8
  u2c.py: upload to copyparty
@@ -62,6 +62,9 @@ else:
62
62
 
63
63
  unicode = str
64
64
 
65
+
66
+ WTF8 = "replace" if PY2 else "surrogateescape"
67
+
65
68
  VT100 = platform.system() != "Windows"
66
69
 
67
70
 
@@ -151,6 +154,7 @@ class HCli(object):
151
154
  self.tls = tls
152
155
  self.verify = ar.te or not ar.td
153
156
  self.conns = []
157
+ self.hconns = []
154
158
  if tls:
155
159
  import ssl
156
160
 
@@ -170,7 +174,7 @@ class HCli(object):
170
174
  "User-Agent": "u2c/%s" % (S_VERSION,),
171
175
  }
172
176
 
173
- def _connect(self):
177
+ def _connect(self, timeout):
174
178
  args = {}
175
179
  if PY37:
176
180
  args["blocksize"] = 1048576
@@ -182,7 +186,7 @@ class HCli(object):
182
186
  if self.ctx:
183
187
  args = {"context": self.ctx}
184
188
 
185
- return C(self.addr, self.port, timeout=999, **args)
189
+ return C(self.addr, self.port, timeout=timeout, **args)
186
190
 
187
191
  def req(self, meth, vpath, hdrs, body=None, ctype=None):
188
192
  hdrs.update(self.base_hdrs)
@@ -195,7 +199,9 @@ class HCli(object):
195
199
  0 if not body else body.len if hasattr(body, "len") else len(body)
196
200
  )
197
201
 
198
- c = self.conns.pop() if self.conns else self._connect()
202
+ # large timeout for handshakes (safededup)
203
+ conns = self.hconns if ctype == MJ else self.conns
204
+ c = conns.pop() if conns else self._connect(999 if ctype == MJ else 128)
199
205
  try:
200
206
  c.request(meth, vpath, body, hdrs)
201
207
  if PY27:
@@ -204,7 +210,7 @@ class HCli(object):
204
210
  rsp = c.getresponse()
205
211
 
206
212
  data = rsp.read()
207
- self.conns.append(c)
213
+ conns.append(c)
208
214
  return rsp.status, data.decode("utf-8")
209
215
  except:
210
216
  c.close()
@@ -228,7 +234,7 @@ class File(object):
228
234
  self.lmod = lmod # type: float
229
235
 
230
236
  self.abs = os.path.join(top, rel) # type: bytes
231
- self.name = self.rel.split(b"/")[-1].decode("utf-8", "replace") # type: str
237
+ self.name = self.rel.split(b"/")[-1].decode("utf-8", WTF8) # type: str
232
238
 
233
239
  # set by get_hashlist
234
240
  self.cids = [] # type: list[tuple[str, int, int]] # [ hash, ofs, sz ]
@@ -267,10 +273,41 @@ class FileSlice(object):
267
273
  raise Exception(9)
268
274
  tlen += clen
269
275
 
270
- self.len = tlen
276
+ self.len = self.tlen = tlen
271
277
  self.cdr = self.car + self.len
272
278
  self.ofs = 0 # type: int
273
- self.f = open(file.abs, "rb", 512 * 1024)
279
+
280
+ self.f = None
281
+ self.seek = self._seek0
282
+ self.read = self._read0
283
+
284
+ def subchunk(self, maxsz, nth):
285
+ if self.tlen <= maxsz:
286
+ return -1
287
+
288
+ if not nth:
289
+ self.car0 = self.car
290
+ self.cdr0 = self.cdr
291
+
292
+ self.car = self.car0 + maxsz * nth
293
+ if self.car >= self.cdr0:
294
+ return -2
295
+
296
+ self.cdr = self.car + min(self.cdr0 - self.car, maxsz)
297
+ self.len = self.cdr - self.car
298
+ self.seek(0)
299
+ return nth
300
+
301
+ def unsub(self):
302
+ self.car = self.car0
303
+ self.cdr = self.cdr0
304
+ self.len = self.tlen
305
+
306
+ def _open(self):
307
+ self.seek = self._seek
308
+ self.read = self._read
309
+
310
+ self.f = open(self.file.abs, "rb", 512 * 1024)
274
311
  self.f.seek(self.car)
275
312
 
276
313
  # https://stackoverflow.com/questions/4359495/what-is-exactly-a-file-like-object-in-python
@@ -282,10 +319,14 @@ class FileSlice(object):
282
319
  except:
283
320
  pass # py27 probably
284
321
 
322
+ def close(self, *a, **ka):
323
+ return # until _open
324
+
285
325
  def tell(self):
286
326
  return self.ofs
287
327
 
288
- def seek(self, ofs, wh=0):
328
+ def _seek(self, ofs, wh=0):
329
+
289
330
  if wh == 1:
290
331
  ofs = self.ofs + ofs
291
332
  elif wh == 2:
@@ -299,12 +340,21 @@ class FileSlice(object):
299
340
  self.ofs = ofs
300
341
  self.f.seek(self.car + ofs)
301
342
 
302
- def read(self, sz):
343
+ def _read(self, sz):
344
+
303
345
  sz = min(sz, self.len - self.ofs)
304
346
  ret = self.f.read(sz)
305
347
  self.ofs += len(ret)
306
348
  return ret
307
349
 
350
+ def _seek0(self, ofs, wh=0):
351
+ self._open()
352
+ return self.seek(ofs, wh)
353
+
354
+ def _read0(self, sz):
355
+ self._open()
356
+ return self.read(sz)
357
+
308
358
 
309
359
  class MTHash(object):
310
360
  def __init__(self, cores):
@@ -557,13 +607,17 @@ def walkdir(err, top, excl, seen):
557
607
  for ap, inf in sorted(statdir(err, top)):
558
608
  if excl.match(ap):
559
609
  continue
560
- yield ap, inf
561
610
  if stat.S_ISDIR(inf.st_mode):
611
+ yield ap, inf
562
612
  try:
563
613
  for x in walkdir(err, ap, excl, seen):
564
614
  yield x
565
615
  except Exception as ex:
566
616
  err.append((ap, str(ex)))
617
+ elif stat.S_ISREG(inf.st_mode):
618
+ yield ap, inf
619
+ else:
620
+ err.append((ap, "irregular filetype 0%o" % (inf.st_mode,)))
567
621
 
568
622
 
569
623
  def walkdirs(err, tops, excl):
@@ -609,11 +663,12 @@ def walkdirs(err, tops, excl):
609
663
 
610
664
  # mostly from copyparty/util.py
611
665
  def quotep(btxt):
666
+ # type: (bytes) -> bytes
612
667
  quot1 = quote(btxt, safe=b"/")
613
668
  if not PY2:
614
669
  quot1 = quot1.encode("ascii")
615
670
 
616
- return quot1.replace(b" ", b"+") # type: ignore
671
+ return quot1.replace(b" ", b"%20") # type: ignore
617
672
 
618
673
 
619
674
  # from copyparty/util.py
@@ -641,7 +696,7 @@ def up2k_chunksize(filesize):
641
696
  while True:
642
697
  for mul in [1, 2]:
643
698
  nchunks = math.ceil(filesize * 1.0 / chunksize)
644
- if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks < 4096):
699
+ if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
645
700
  return chunksize
646
701
 
647
702
  chunksize += stepsize
@@ -720,7 +775,7 @@ def handshake(ar, file, search):
720
775
  url = file.url
721
776
  else:
722
777
  if b"/" in file.rel:
723
- url = quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8", "replace")
778
+ url = quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8")
724
779
  else:
725
780
  url = ""
726
781
  url = ar.vtop + url
@@ -766,15 +821,15 @@ def handshake(ar, file, search):
766
821
  if search:
767
822
  return r["hits"], False
768
823
 
769
- file.url = r["purl"]
824
+ file.url = quotep(r["purl"].encode("utf-8", WTF8)).decode("utf-8")
770
825
  file.name = r["name"]
771
826
  file.wark = r["wark"]
772
827
 
773
828
  return r["hash"], r["sprs"]
774
829
 
775
830
 
776
- def upload(fsl, stats):
777
- # type: (FileSlice, str) -> None
831
+ def upload(fsl, stats, maxsz):
832
+ # type: (FileSlice, str, int) -> None
778
833
  """upload a range of file data, defined by one or more `cid` (chunk-hash)"""
779
834
 
780
835
  ctxt = fsl.cids[0]
@@ -792,21 +847,34 @@ def upload(fsl, stats):
792
847
  if stats:
793
848
  headers["X-Up2k-Stat"] = stats
794
849
 
850
+ nsub = 0
795
851
  try:
796
- sc, txt = web.req("POST", fsl.file.url, headers, fsl, MO)
797
-
798
- if sc == 400:
799
- if (
800
- "already being written" in txt
801
- or "already got that" in txt
802
- or "only sibling chunks" in txt
803
- ):
804
- fsl.file.nojoin = 1
805
-
806
- if sc >= 400:
807
- raise Exception("http %s: %s" % (sc, txt))
852
+ while nsub != -1:
853
+ nsub = fsl.subchunk(maxsz, nsub)
854
+ if nsub == -2:
855
+ return
856
+ if nsub >= 0:
857
+ headers["X-Up2k-Subc"] = str(maxsz * nsub)
858
+ headers.pop(CLEN, None)
859
+ nsub += 1
860
+
861
+ sc, txt = web.req("POST", fsl.file.url, headers, fsl, MO)
862
+
863
+ if sc == 400:
864
+ if (
865
+ "already being written" in txt
866
+ or "already got that" in txt
867
+ or "only sibling chunks" in txt
868
+ ):
869
+ fsl.file.nojoin = 1
870
+
871
+ if sc >= 400:
872
+ raise Exception("http %s: %s" % (sc, txt))
808
873
  finally:
809
- fsl.f.close()
874
+ if fsl.f:
875
+ fsl.f.close()
876
+ if nsub != -1:
877
+ fsl.unsub()
810
878
 
811
879
 
812
880
  class Ctl(object):
@@ -938,7 +1006,7 @@ class Ctl(object):
938
1006
  print(" %d up %s" % (ncs - nc, cid))
939
1007
  stats = "%d/0/0/%d" % (nf, self.nfiles - nf)
940
1008
  fslice = FileSlice(file, [cid])
941
- upload(fslice, stats)
1009
+ upload(fslice, stats, self.ar.szm)
942
1010
 
943
1011
  print(" ok!")
944
1012
  if file.recheck:
@@ -1057,7 +1125,7 @@ class Ctl(object):
1057
1125
  print(" ls ~{0}".format(srd))
1058
1126
  zt = (
1059
1127
  self.ar.vtop,
1060
- quotep(rd.replace(b"\\", b"/")).decode("utf-8", "replace"),
1128
+ quotep(rd.replace(b"\\", b"/")).decode("utf-8"),
1061
1129
  )
1062
1130
  sc, txt = web.req("GET", "%s%s?ls&lt&dots" % zt, {})
1063
1131
  if sc >= 400:
@@ -1066,7 +1134,7 @@ class Ctl(object):
1066
1134
  j = json.loads(txt)
1067
1135
  for f in j["dirs"] + j["files"]:
1068
1136
  rfn = f["href"].split("?")[0].rstrip("/")
1069
- ls[unquote(rfn.encode("utf-8", "replace"))] = f
1137
+ ls[unquote(rfn.encode("utf-8", WTF8))] = f
1070
1138
  except Exception as ex:
1071
1139
  print(" mkdir ~{0} ({1})".format(srd, ex))
1072
1140
 
@@ -1080,7 +1148,7 @@ class Ctl(object):
1080
1148
  lnodes = [x.split(b"/")[-1] for x in zls]
1081
1149
  bnames = [x for x in ls if x not in lnodes and x != b".hist"]
1082
1150
  vpath = self.ar.url.split("://")[-1].split("/", 1)[-1]
1083
- names = [x.decode("utf-8", "replace") for x in bnames]
1151
+ names = [x.decode("utf-8", WTF8) for x in bnames]
1084
1152
  locs = [vpath + srd + "/" + x for x in names]
1085
1153
  while locs:
1086
1154
  req = locs
@@ -1286,7 +1354,7 @@ class Ctl(object):
1286
1354
  self._check_if_done()
1287
1355
  continue
1288
1356
 
1289
- njoin = (self.ar.sz * 1024 * 1024) // chunksz
1357
+ njoin = self.ar.sz // chunksz
1290
1358
  cs = hs[:]
1291
1359
  while cs:
1292
1360
  fsl = FileSlice(file, cs[:1])
@@ -1338,7 +1406,7 @@ class Ctl(object):
1338
1406
  )
1339
1407
 
1340
1408
  try:
1341
- upload(fsl, stats)
1409
+ upload(fsl, stats, self.ar.szm)
1342
1410
  except Exception as ex:
1343
1411
  t = "upload failed, retrying: %s #%s+%d (%s)\n"
1344
1412
  eprint(t % (file.name, cids[0][:8], len(cids) - 1, ex))
@@ -1427,6 +1495,7 @@ source file/folder selection uses rsync syntax, meaning that:
1427
1495
  ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
1428
1496
  ap.add_argument("-J", type=int, metavar="CORES", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
1429
1497
  ap.add_argument("--sz", type=int, metavar="MiB", default=64, help="try to make each POST this big")
1498
+ ap.add_argument("--szm", type=int, metavar="MiB", default=96, help="max size of each POST (default is cloudflare max)")
1430
1499
  ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
1431
1500
  ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
1432
1501
  ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
@@ -1454,6 +1523,9 @@ source file/folder selection uses rsync syntax, meaning that:
1454
1523
  if ar.dr:
1455
1524
  ar.ow = True
1456
1525
 
1526
+ ar.sz *= 1024 * 1024
1527
+ ar.szm *= 1024 * 1024
1528
+
1457
1529
  ar.x = "|".join(ar.x or [])
1458
1530
 
1459
1531
  setattr(ar, "wlist", ar.url == "-")
Binary file
Binary file
copyparty/web/ui.css.gz CHANGED
Binary file
copyparty/web/up2k.js.gz CHANGED
Binary file
copyparty/web/util.js.gz CHANGED
Binary file
Binary file
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: copyparty
3
- Version: 1.15.7
3
+ Version: 1.15.9
4
4
  Summary: Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
5
5
  Author-email: ed <copyparty@ocv.me>
6
6
  License: MIT
@@ -101,6 +101,7 @@ turn almost any device into a file server with resumable uploads/downloads using
101
101
  * [file manager](#file-manager) - cut/paste, rename, and delete files/folders (if you have permission)
102
102
  * [shares](#shares) - share a file or folder by creating a temporary link
103
103
  * [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
104
+ * [rss feeds](#rss-feeds) - monitor a folder with your RSS reader
104
105
  * [media player](#media-player) - plays almost every audio format there is
105
106
  * [audio equalizer](#audio-equalizer) - and [dynamic range compressor](https://en.wikipedia.org/wiki/Dynamic_range_compression)
106
107
  * [fix unreliable playback on android](#fix-unreliable-playback-on-android) - due to phone / app settings
@@ -273,7 +274,7 @@ also see [comparison to similar software](./docs/versus.md)
273
274
  * upload
274
275
  * ☑ basic: plain multipart, ie6 support
275
276
  * ☑ [up2k](#uploading): js, resumable, multithreaded
276
- * **no filesize limit!** ...unless you use Cloudflare, then it's 383.9 GiB
277
+ * **no filesize limit!** even on Cloudflare
277
278
  * ☑ stash: simple PUT filedropper
278
279
  * ☑ filename randomizer
279
280
  * ☑ write-only folders
@@ -708,7 +709,7 @@ up2k has several advantages:
708
709
  * uploads resume if you reboot your browser or pc, just upload the same files again
709
710
  * server detects any corruption; the client reuploads affected chunks
710
711
  * the client doesn't upload anything that already exists on the server
711
- * no filesize limit unless imposed by a proxy, for example Cloudflare, which blocks uploads over 383.9 GiB
712
+ * no filesize limit, even when a proxy limits the request size (for example Cloudflare)
712
713
  * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
713
714
  * the last-modified timestamp of the file is preserved
714
715
 
@@ -744,6 +745,8 @@ note that since up2k has to read each file twice, `[🎈] bup` can *theoreticall
744
745
 
745
746
  if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
746
747
 
748
+ if the server is behind a proxy which imposes a request-size limit, you can configure up2k to sneak below the limit with server-option `--u2sz` (the default is 96 MiB to support Cloudflare)
749
+
747
750
 
748
751
  ### file-search
749
752
 
@@ -897,6 +900,30 @@ or a mix of both:
897
900
  the metadata keys you can use in the format field are the ones in the file-browser table header (whatever is collected with `-mte` and `-mtp`)
898
901
 
899
902
 
903
+ ## rss feeds
904
+
905
+ monitor a folder with your RSS reader , optionally recursive
906
+
907
+ must be enabled per-volume with volflag `rss` or globally with `--rss`
908
+
909
+ the feed includes itunes metadata for use with podcast readers such as [AntennaPod](https://antennapod.org/)
910
+
911
+ a feed example: https://cd.ocv.me/a/d2/d22/?rss&fext=mp3
912
+
913
+ url parameters:
914
+
915
+ * `pw=hunter2` for password auth
916
+ * `recursive` to also include subfolders
917
+ * `title=foo` changes the feed title (default: folder name)
918
+ * `fext=mp3,opus` only include mp3 and opus files (default: all)
919
+ * `nf=30` only show the first 30 results (default: 250)
920
+ * `sort=m` sort by mtime (file last-modified), newest first (default)
921
+ * `u` = upload-time; NOTE: non-uploaded files have upload-time `0`
922
+ * `n` = filename
923
+ * `a` = filesize
924
+ * uppercase = reverse-sort; `M` = oldest file first
925
+
926
+
900
927
  ## media player
901
928
 
902
929
  plays almost every audio format there is (if the server has FFmpeg installed for on-demand transcoding)
@@ -1,22 +1,22 @@
1
1
  copyparty/__init__.py,sha256=Chqw7uXX4r_-a2p6-xthrrqVHFI4aZdW45sWU7UvqeE,2597
2
- copyparty/__main__.py,sha256=jPcqdsSJZP6AMoPygb6t8PwxmLXGK3jUeUvt28JcnyE,110130
3
- copyparty/__version__.py,sha256=Z9byW4PqObsayHm5sQR8iTi3mFCP5dkS0YnU-6VoRT4,258
4
- copyparty/authsrv.py,sha256=clsE8whf32_eIRES1r7VvsVkc4OiJCjiBEJM6pYkwwI,98670
2
+ copyparty/__main__.py,sha256=rx8OlvcX3M-SQOvkrYT9C-HN7GmlPUx5p_2Vqv_LEo4,110949
3
+ copyparty/__version__.py,sha256=PMcIJPKN9PvxPb2mIFEiFAsp9HLavA8qaM4ckTc7cNE,258
4
+ copyparty/authsrv.py,sha256=Iw_4lJUhRU9q3qCQucGWRtJOFKYrYd5omL5nmwGvw-k,98979
5
5
  copyparty/broker_mp.py,sha256=jsHUM2BSfRVRyZT869iPCqYEHSqedk6VkwvygZwbEZE,4017
6
6
  copyparty/broker_mpw.py,sha256=PYFgQfssOCfdI6qayW1ZjO1j1-7oez094muhYMbPOz0,3339
7
7
  copyparty/broker_thr.py,sha256=MXrwjusP0z1LPURUhi5jx_TL3jrXhYcDrJPDSKu6EEU,1705
8
8
  copyparty/broker_util.py,sha256=76mfnFOpX1gUUvtjm8UQI7jpTIaVINX10QonM-B7ggc,1680
9
9
  copyparty/cert.py,sha256=0ZAPeXeMR164vWn9GQU3JDKooYXEq_NOQkDeg543ivg,8009
10
- copyparty/cfg.py,sha256=33nLatBUmzRFKQ4KpoQei3ZY6EqRrlaHpQnvCNFXcHI,10112
10
+ copyparty/cfg.py,sha256=E9iBGNjIUrDAPLFRgKsVOmAknP9bDE27xh0gkmNdH1s,10127
11
11
  copyparty/dxml.py,sha256=lZpg-kn-kQsXRtNY1n6fRaS-b7uXzMCyv8ovKnhZcZc,1548
12
12
  copyparty/fsutil.py,sha256=5CshJWO7CflfaRRNOb3JxghUH7W5rmS_HWNmKfx42MM,4538
13
13
  copyparty/ftpd.py,sha256=G_h1urfIikzfCWGXnW9p-rioWdNM_Je6vWYq0-QSbC8,17580
14
- copyparty/httpcli.py,sha256=PIlYJa-1QTzsASD9jRbRJGsn8vAUNSg15qJEigcs9cs,192083
14
+ copyparty/httpcli.py,sha256=irIxsAI0KpNvxoenK16zkezttl4sUSNUB59yBr8L6VA,199053
15
15
  copyparty/httpconn.py,sha256=mQSgljh0Q-jyWjF4tQLrHbRKRe9WKl19kGqsGMsJpWo,6880
16
- copyparty/httpsrv.py,sha256=k0xSadmUNSyUAWlZdOMFdsBP1fQK50Ptv7O6O4jFy2Q,17182
16
+ copyparty/httpsrv.py,sha256=d_UiGnQKniBoEV68lNFgnYm-byda7uj56mFf-YC7piI,17223
17
17
  copyparty/ico.py,sha256=eWSxEae4wOCfheHl-m-wchYvFRAR_97kJDb4NGaB-Z8,3561
18
18
  copyparty/mdns.py,sha256=vC078llnL1v0pvL3mnwacuStFHPJUQuxo9Opj-IbHL4,18155
19
- copyparty/metrics.py,sha256=aV09nntEmKMIyde8xoPtj1ehDOQVQOHchRF4uMMNzqM,8855
19
+ copyparty/metrics.py,sha256=-1Rkk44gBh_1YJbdzGZHaqR4pEwkbno6fSdsRb5wDIk,8865
20
20
  copyparty/mtag.py,sha256=8WGjEn0T0Ri9ww1yBpLUnFHZiTQMye1BMXL6SkK3MRo,18893
21
21
  copyparty/multicast.py,sha256=Ha27l2oATEa-Qo2WOzkeRgjAm6G_YDCfbVJWR-ao2UE,12319
22
22
  copyparty/pwhash.py,sha256=AdLMLyIi2IDhGtbKIQOswKUxWvO7ARYYRF_ThsryOoc,4124
@@ -30,9 +30,9 @@ copyparty/tcpsrv.py,sha256=l_vb9FoF0AJur0IoqHNUSBDqMgBO_MRUZeDszi1UNfY,19881
30
30
  copyparty/tftpd.py,sha256=jZbf2JpeJmkuQWJErmAPG-dKhtYNvIUHbkAgodSXw9Y,13582
31
31
  copyparty/th_cli.py,sha256=o6FMkerYvAXS455z3DUossVztu_nzFlYSQhs6qN6Jt8,4636
32
32
  copyparty/th_srv.py,sha256=hI9wY1E_9N9Cgqvtr8zADeVqqiLGTiTdAnYAA7WFvJw,29346
33
- copyparty/u2idx.py,sha256=JjgqwgJBNj6sTn4PJfuqM3VEHqlmoyGC5bk4_92K2h0,13414
34
- copyparty/up2k.py,sha256=PSVSdG00wC76mi0E7HF1WBFco4lujpSCTTtSzUusPVM,165096
35
- copyparty/util.py,sha256=Vj8F50G0taqNd7c8TQd56IRbRZGFE1r0wnnrymbk7j0,92349
33
+ copyparty/u2idx.py,sha256=HLO49L1zmpJtBcJiXgD12a6pAlQdnf2pFelHMA7habw,13462
34
+ copyparty/up2k.py,sha256=2K21-Scz4c3_Y9MzAPmvq3_LEOIkFnx7KLR6hqJkP18,165419
35
+ copyparty/util.py,sha256=eJU7a5O-bZf_MEplCVmLoS3r5mjPC3-2khqZVTYGw-c,92653
36
36
  copyparty/bos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
37
  copyparty/bos/bos.py,sha256=Wb7eWsXJgR5AFlBR9ZOyKrLTwy-Kct9RrGiOu4Jo37Y,1622
38
38
  copyparty/bos/path.py,sha256=yEjCq2ki9CvxA5sCT8pS0keEXwugs0ZeUyUhdBziOCI,777
@@ -54,10 +54,10 @@ copyparty/stolen/ifaddr/__init__.py,sha256=vpREjAyPubr5s1NJi91icXV3q1o4DrKAvHABw
54
54
  copyparty/stolen/ifaddr/_posix.py,sha256=-67NdfGrCktfQPakT2fLbjl2U00QMvyBGkSvrUuTOrU,2626
55
55
  copyparty/stolen/ifaddr/_shared.py,sha256=uNC4SdEIgdSLKvuUzsf1aM-H1Xrc_9mpLoOT43YukGs,6206
56
56
  copyparty/stolen/ifaddr/_win32.py,sha256=EE-QyoBgeB7lYQ6z62VjXNaRozaYfCkaJBHGNA8QtZM,4026
57
- copyparty/web/baguettebox.js.gz,sha256=4dS8-r4si84ca71l98672ahnRI86Aq95MU-bc5knykk,7962
57
+ copyparty/web/baguettebox.js.gz,sha256=YIaxFDsubJfGIdzzxA-cL6GwJVmpWZyaPhW9hHcOIIw,7964
58
58
  copyparty/web/browser.css.gz,sha256=4bAS9Xkl2fflhaxRSRSVoYQcpXsg1mCWxsYjId7phbU,11610
59
59
  copyparty/web/browser.html,sha256=ISpfvWEawufJCYZIqvuXiyUgiXgjmOTtScz4zrEaypI,4870
60
- copyparty/web/browser.js.gz,sha256=0WhsCM1Kr0PFryxz7hF31apLgvsJac4jWy8_ePfp_40,84989
60
+ copyparty/web/browser.js.gz,sha256=7rubbEoqFlNn7FPF0mz3L2LQeWuPzw95MYpIaZmcczE,84985
61
61
  copyparty/web/browser2.html,sha256=NRUZ08GH-e2YcGXcoz0UjYg6JIVF42u4IMX4HHwWTmg,1587
62
62
  copyparty/web/cf.html,sha256=lJThtNFNAQT1ClCHHlivAkDGE0LutedwopXD62Z8Nys,589
63
63
  copyparty/web/dbg-audio.js.gz,sha256=Ma-KZtK8LnmiwNvNKFKXMPYl_Nn_3U7GsJ6-DRWC2HE,688
@@ -79,13 +79,13 @@ copyparty/web/splash.html,sha256=pUbsso_W3Q7bso8fy8qxh-fHDrrLm39mBBTIlTeH63w,523
79
79
  copyparty/web/splash.js.gz,sha256=Xoccku-2vE3tABo-88q3Cl4koHs_AE76T8QvMy4u6T8,2540
80
80
  copyparty/web/svcs.html,sha256=P5YZimYLeQMT0uz6u3clQSNZRc5Zs0Ok-ffcbcGSYuc,11762
81
81
  copyparty/web/svcs.js.gz,sha256=k81ZvZ3I-f4fMHKrNGGOgOlvXnCBz0mVjD-8mieoWCA,520
82
- copyparty/web/ui.css.gz,sha256=dTAXEPgWB83CkYXlP4hWq18BrLFzRlYwxoCoM5rp-Yw,2786
83
- copyparty/web/up2k.js.gz,sha256=t6mUaIYN8lrEJoWO_-2MZ9nIuEplimQgCqdWSsHnuGA,22811
84
- copyparty/web/util.js.gz,sha256=uh_NAVPiMcOCTy9oxUiDKmKh1GokTKBPe3FYNMMUYlM,14781
85
- copyparty/web/w.hash.js.gz,sha256=7wP9EZQNXQxwZnCCFUVsi_-6TM9PLZJeZ9krutXRRj8,1060
82
+ copyparty/web/ui.css.gz,sha256=wloSacrHgP722hy4XiOvVY2GI9-V4zvfvzu84LLWS_o,2779
83
+ copyparty/web/up2k.js.gz,sha256=lGR1Xb0RkIZ1eHmncsSwWRuFc6FC2rZalvjo3oNTV1s,23291
84
+ copyparty/web/util.js.gz,sha256=NvjPYhIa0-C_NhUyW-Ra-XinUCRjj8G3pYq1zJHYWEk,14805
85
+ copyparty/web/w.hash.js.gz,sha256=l3GpSJD6mcU-1CRWkIj7PybgbjlfSr8oeO3vortIrQk,1105
86
86
  copyparty/web/a/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
87
87
  copyparty/web/a/partyfuse.py,sha256=fa9bBYNJHvtWpNVjQyyFzx6JOK7MJL1u0zj80PBYQKs,27960
88
- copyparty/web/a/u2c.py,sha256=TkLmQL_3lwus0HI-m4cuP8E9qanfB8U9xGvW7zEJ8ho,47576
88
+ copyparty/web/a/u2c.py,sha256=ZmLcGuOWB66ZkAMpJBiR1Xpa75PgpzdFRTt3GGVCorc,49533
89
89
  copyparty/web/a/webdav-cfg.bat,sha256=Y4NoGZlksAIg4cBMb7KdJrpKC6Nx97onaTl6yMjaimk,1449
90
90
  copyparty/web/dd/2.png,sha256=gJ14XFPzaw95L6z92fSq9eMPikSQyu-03P1lgiGe0_I,258
91
91
  copyparty/web/dd/3.png,sha256=4lho8Koz5tV7jJ4ODo6GMTScZfkqsT05yp48EDFIlyg,252
@@ -106,9 +106,9 @@ copyparty/web/deps/prismd.css.gz,sha256=ObUlksQVr-OuYlTz-I4B23TeBg2QDVVGRnWBz8cV
106
106
  copyparty/web/deps/scp.woff2,sha256=w99BDU5i8MukkMEL-iW0YO9H4vFFZSPWxbkH70ytaAg,8612
107
107
  copyparty/web/deps/sha512.ac.js.gz,sha256=lFZaCLumgWxrvEuDr4bqdKHsqjX82AbVAb7_F45Yk88,7033
108
108
  copyparty/web/deps/sha512.hw.js.gz,sha256=vqoXeracj-99Z5MfY3jK2N4WiSzYQdfjy0RnUlQDhSU,8110
109
- copyparty-1.15.7.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
110
- copyparty-1.15.7.dist-info/METADATA,sha256=DXlK1tdi4bTueoZXTbr33g_wqCpjko4sTPWpTeXXfGs,138774
111
- copyparty-1.15.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
112
- copyparty-1.15.7.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
113
- copyparty-1.15.7.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
114
- copyparty-1.15.7.dist-info/RECORD,,
109
+ copyparty-1.15.9.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
110
+ copyparty-1.15.9.dist-info/METADATA,sha256=1GMM2x8ZO0z3TgiQgCB6xEiiahXsn3PdeiA7Uni_D_4,139807
111
+ copyparty-1.15.9.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
112
+ copyparty-1.15.9.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
113
+ copyparty-1.15.9.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
114
+ copyparty-1.15.9.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.1.0)
2
+ Generator: setuptools (75.2.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5