copyparty 1.15.6__py3-none-any.whl → 1.15.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
copyparty/__main__.py CHANGED
@@ -1009,7 +1009,7 @@ def add_upload(ap):
1009
1009
  ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
1010
1010
  ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m-1\033[0m] = forbidden/always-off, [\033[32m0\033[0m] = default-off and warn if enabled, [\033[32m1\033[0m] = default-off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
1011
1011
  ap2.add_argument("--u2j", metavar="JOBS", type=int, default=2, help="web-client: number of file chunks to upload in parallel; 1 or 2 is good for low-latency (same-country) connections, 4-8 for android clients, 16 for cross-atlantic (max=64)")
1012
- ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for this size. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
1012
+ ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for \033[33mdefault\033[0m, and never exceed \033[33mmax\033[0m. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
1013
1013
  ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
1014
1014
  ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
1015
1015
 
@@ -1079,6 +1079,7 @@ def add_auth(ap):
1079
1079
  ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)")
1080
1080
  ap2.add_argument("--ses-len", metavar="CHARS", type=int, default=20, help="session key length; default is 120 bits ((20//4)*4*6)")
1081
1081
  ap2.add_argument("--no-ses", action="store_true", help="disable sessions; use plaintext passwords in cookies")
1082
+ ap2.add_argument("--ipu", metavar="CIDR=USR", type=u, action="append", help="users with IP matching \033[33mCIDR\033[0m are auto-authenticated as username \033[33mUSR\033[0m; example: [\033[32m172.16.24.0/24=dave]")
1082
1083
 
1083
1084
 
1084
1085
  def add_chpw(ap):
@@ -1469,6 +1470,7 @@ def add_debug(ap):
1469
1470
  ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to \033[33m--bf-nc\033[0m and \033[33m--bf-dir\033[0m")
1470
1471
  ap2.add_argument("--bf-nc", metavar="NUM", type=int, default=200, help="bak-flips: stop if there's more than \033[33mNUM\033[0m files at \033[33m--kf-dir\033[0m already; default: 6.3 GiB max (200*32M)")
1471
1472
  ap2.add_argument("--bf-dir", metavar="PATH", type=u, default="bf", help="bak-flips: store corrupted chunks at \033[33mPATH\033[0m; default: folder named 'bf' wherever copyparty was started")
1473
+ ap2.add_argument("--bf-log", metavar="PATH", type=u, default="", help="bak-flips: log corruption info to a textfile at \033[33mPATH\033[0m")
1472
1474
 
1473
1475
 
1474
1476
  # fmt: on
copyparty/__version__.py CHANGED
@@ -1,8 +1,8 @@
1
1
  # coding: utf-8
2
2
 
3
- VERSION = (1, 15, 6)
3
+ VERSION = (1, 15, 8)
4
4
  CODENAME = "fill the drives"
5
- BUILD_DT = (2024, 10, 12)
5
+ BUILD_DT = (2024, 10, 16)
6
6
 
7
7
  S_VERSION = ".".join(map(str, VERSION))
8
8
  S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
copyparty/authsrv.py CHANGED
@@ -59,6 +59,7 @@ if PY2:
59
59
  LEELOO_DALLAS = "leeloo_dallas"
60
60
 
61
61
  SEE_LOG = "see log for details"
62
+ SEESLOG = " (see serverlog for details)"
62
63
  SSEELOG = " ({})".format(SEE_LOG)
63
64
  BAD_CFG = "invalid config; {}".format(SEE_LOG)
64
65
  SBADCFG = " ({})".format(BAD_CFG)
copyparty/ftpd.py CHANGED
@@ -72,6 +72,7 @@ class FtpAuth(DummyAuthorizer):
72
72
  else:
73
73
  raise AuthenticationFailed("banned")
74
74
 
75
+ args = self.hub.args
75
76
  asrv = self.hub.asrv
76
77
  uname = "*"
77
78
  if username != "anonymous":
@@ -82,6 +83,9 @@ class FtpAuth(DummyAuthorizer):
82
83
  uname = zs
83
84
  break
84
85
 
86
+ if args.ipu and uname == "*":
87
+ uname = args.ipu_iu[args.ipu_nm.map(ip)]
88
+
85
89
  if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
86
90
  g = self.hub.gpwd
87
91
  if g.lim:
copyparty/httpcli.py CHANGED
@@ -584,6 +584,9 @@ class HttpCli(object):
584
584
  or "*"
585
585
  )
586
586
 
587
+ if self.args.ipu and self.uname == "*":
588
+ self.uname = self.conn.ipu_iu[self.conn.ipu_nm.map(self.ip)]
589
+
587
590
  self.rvol = self.asrv.vfs.aread[self.uname]
588
591
  self.wvol = self.asrv.vfs.awrite[self.uname]
589
592
  self.avol = self.asrv.vfs.aadmin[self.uname]
@@ -1871,7 +1874,7 @@ class HttpCli(object):
1871
1874
  f, fn = ren_open(fn, *open_a, **params)
1872
1875
  try:
1873
1876
  path = os.path.join(fdir, fn)
1874
- post_sz, sha_hex, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
1877
+ post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp)
1875
1878
  finally:
1876
1879
  f.close()
1877
1880
 
@@ -2023,13 +2026,32 @@ class HttpCli(object):
2023
2026
  return True
2024
2027
 
2025
2028
  def bakflip(
2026
- self, f , ofs , sz , sha , flags
2029
+ self,
2030
+ f ,
2031
+ ap ,
2032
+ ofs ,
2033
+ sz ,
2034
+ good_sha ,
2035
+ bad_sha ,
2036
+ flags ,
2027
2037
  ) :
2038
+ now = time.time()
2039
+ t = "bad-chunk: %.3f %s %s %d %s %s %s"
2040
+ t = t % (now, bad_sha, good_sha, ofs, self.ip, self.uname, ap)
2041
+ self.log(t, 5)
2042
+
2043
+ if self.args.bf_log:
2044
+ try:
2045
+ with open(self.args.bf_log, "ab+") as f2:
2046
+ f2.write((t + "\n").encode("utf-8", "replace"))
2047
+ except Exception as ex:
2048
+ self.log("append %s failed: %r" % (self.args.bf_log, ex))
2049
+
2028
2050
  if not self.args.bak_flips or self.args.nw:
2029
2051
  return
2030
2052
 
2031
2053
  sdir = self.args.bf_dir
2032
- fp = os.path.join(sdir, sha)
2054
+ fp = os.path.join(sdir, bad_sha)
2033
2055
  if bos.path.exists(fp):
2034
2056
  return self.log("no bakflip; have it", 6)
2035
2057
 
@@ -2315,7 +2337,7 @@ class HttpCli(object):
2315
2337
  broker = self.conn.hsrv.broker
2316
2338
  x = broker.ask("up2k.handle_chunks", ptop, wark, chashes)
2317
2339
  response = x.get()
2318
- chashes, chunksize, cstarts, path, lastmod, sprs = response
2340
+ chashes, chunksize, cstarts, path, lastmod, fsize, sprs = response
2319
2341
  maxsize = chunksize * len(chashes)
2320
2342
  cstart0 = cstarts[0]
2321
2343
  locked = chashes # remaining chunks to be received in this request
@@ -2323,6 +2345,50 @@ class HttpCli(object):
2323
2345
  num_left = -1 # num chunks left according to most recent up2k release
2324
2346
  treport = time.time() # ratelimit up2k reporting to reduce overhead
2325
2347
 
2348
+ if "x-up2k-subc" in self.headers:
2349
+ sc_ofs = int(self.headers["x-up2k-subc"])
2350
+ chash = chashes[0]
2351
+
2352
+ u2sc = self.conn.hsrv.u2sc
2353
+ try:
2354
+ sc_pofs, hasher = u2sc[chash]
2355
+ if not sc_ofs:
2356
+ t = "client restarted the chunk; forgetting subchunk offset %d"
2357
+ self.log(t % (sc_pofs,))
2358
+ raise Exception()
2359
+ except:
2360
+ sc_pofs = 0
2361
+ hasher = hashlib.sha512()
2362
+
2363
+ et = "subchunk protocol error; resetting chunk "
2364
+ if sc_pofs != sc_ofs:
2365
+ u2sc.pop(chash, None)
2366
+ t = "%s[%s]: the expected resume-point was %d, not %d"
2367
+ raise Pebkac(400, t % (et, chash, sc_pofs, sc_ofs))
2368
+ if len(cstarts) > 1:
2369
+ u2sc.pop(chash, None)
2370
+ t = "%s[%s]: only a single subchunk can be uploaded in one request; you are sending %d chunks"
2371
+ raise Pebkac(400, t % (et, chash, len(cstarts)))
2372
+ csize = min(chunksize, fsize - cstart0[0])
2373
+ cstart0[0] += sc_ofs # also sets cstarts[0][0]
2374
+ sc_next_ofs = sc_ofs + postsize
2375
+ if sc_next_ofs > csize:
2376
+ u2sc.pop(chash, None)
2377
+ t = "%s[%s]: subchunk offset (%d) plus postsize (%d) exceeds chunksize (%d)"
2378
+ raise Pebkac(400, t % (et, chash, sc_ofs, postsize, csize))
2379
+ else:
2380
+ final_subchunk = sc_next_ofs == csize
2381
+ t = "subchunk %s %d:%d/%d %s"
2382
+ zs = "END" if final_subchunk else ""
2383
+ self.log(t % (chash[:15], sc_ofs, sc_next_ofs, csize, zs), 6)
2384
+ if final_subchunk:
2385
+ u2sc.pop(chash, None)
2386
+ else:
2387
+ u2sc[chash] = (sc_next_ofs, hasher)
2388
+ else:
2389
+ hasher = None
2390
+ final_subchunk = True
2391
+
2326
2392
  try:
2327
2393
  if self.args.nw:
2328
2394
  path = os.devnull
@@ -2353,11 +2419,15 @@ class HttpCli(object):
2353
2419
  reader = read_socket(
2354
2420
  self.sr, self.args.s_rd_sz, min(remains, chunksize)
2355
2421
  )
2356
- post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
2422
+ post_sz, _, sha_b64 = hashcopy(
2423
+ reader, f, hasher, 0, self.args.s_wr_slp
2424
+ )
2357
2425
 
2358
- if sha_b64 != chash:
2426
+ if sha_b64 != chash and final_subchunk:
2359
2427
  try:
2360
- self.bakflip(f, cstart[0], post_sz, sha_b64, vfs.flags)
2428
+ self.bakflip(
2429
+ f, path, cstart[0], post_sz, chash, sha_b64, vfs.flags
2430
+ )
2361
2431
  except:
2362
2432
  self.log("bakflip failed: " + min_ex())
2363
2433
 
@@ -2385,7 +2455,8 @@ class HttpCli(object):
2385
2455
 
2386
2456
  # be quick to keep the tcp winsize scale;
2387
2457
  # if we can't confirm rn then that's fine
2388
- written.append(chash)
2458
+ if final_subchunk:
2459
+ written.append(chash)
2389
2460
  now = time.time()
2390
2461
  if now - treport < 1:
2391
2462
  continue
@@ -2773,7 +2844,7 @@ class HttpCli(object):
2773
2844
  tabspath = os.path.join(fdir, tnam)
2774
2845
  self.log("writing to {}".format(tabspath))
2775
2846
  sz, sha_hex, sha_b64 = hashcopy(
2776
- p_data, f, self.args.s_wr_slp, max_sz
2847
+ p_data, f, None, max_sz, self.args.s_wr_slp
2777
2848
  )
2778
2849
  if sz == 0:
2779
2850
  raise Pebkac(400, "empty files in post")
@@ -3103,7 +3174,7 @@ class HttpCli(object):
3103
3174
  wunlink(self.log, fp, vfs.flags)
3104
3175
 
3105
3176
  with open(fsenc(fp), "wb", self.args.iobuf) as f:
3106
- sz, sha512, _ = hashcopy(p_data, f, self.args.s_wr_slp)
3177
+ sz, sha512, _ = hashcopy(p_data, f, None, 0, self.args.s_wr_slp)
3107
3178
 
3108
3179
  if lim:
3109
3180
  lim.nup(self.ip)
copyparty/httpconn.py CHANGED
@@ -56,6 +56,8 @@ class HttpConn(object):
56
56
  self.asrv = hsrv.asrv # mypy404
57
57
  self.u2fh = hsrv.u2fh # mypy404
58
58
  self.pipes = hsrv.pipes # mypy404
59
+ self.ipu_iu = hsrv.ipu_iu
60
+ self.ipu_nm = hsrv.ipu_nm
59
61
  self.ipa_nm = hsrv.ipa_nm
60
62
  self.xff_nm = hsrv.xff_nm
61
63
  self.xff_lan = hsrv.xff_lan # type: ignore
copyparty/httpsrv.py CHANGED
@@ -1,6 +1,7 @@
1
1
  # coding: utf-8
2
2
  from __future__ import print_function, unicode_literals
3
3
 
4
+ import hashlib
4
5
  import math
5
6
  import os
6
7
  import re
@@ -69,6 +70,7 @@ from .util import (
69
70
  build_netmap,
70
71
  has_resource,
71
72
  ipnorm,
73
+ load_ipu,
72
74
  load_resource,
73
75
  min_ex,
74
76
  shut_socket,
@@ -140,6 +142,7 @@ class HttpSrv(object):
140
142
  self.t_periodic = None
141
143
 
142
144
  self.u2fh = FHC()
145
+ self.u2sc = {}
143
146
  self.pipes = CachedDict(0.2)
144
147
  self.metrics = Metrics(self)
145
148
  self.nreq = 0
@@ -171,6 +174,11 @@ class HttpSrv(object):
171
174
  self.j2 = {x: env.get_template(x + ".html") for x in jn}
172
175
  self.prism = has_resource(self.E, "web/deps/prism.js.gz")
173
176
 
177
+ if self.args.ipu:
178
+ self.ipu_iu, self.ipu_nm = load_ipu(self.log, self.args.ipu)
179
+ else:
180
+ self.ipu_iu = self.ipu_nm = None
181
+
174
182
  self.ipa_nm = build_netmap(self.args.ipa)
175
183
  self.xff_nm = build_netmap(self.args.xff_src)
176
184
  self.xff_lan = build_netmap("lan")
copyparty/svchub.py CHANGED
@@ -54,6 +54,7 @@ from .util import (
54
54
  alltrace,
55
55
  ansi_re,
56
56
  build_netmap,
57
+ load_ipu,
57
58
  min_ex,
58
59
  mp,
59
60
  odfusion,
@@ -215,6 +216,11 @@ class SvcHub(object):
215
216
  noch.update([x for x in zsl if x])
216
217
  args.chpw_no = noch
217
218
 
219
+ if args.ipu:
220
+ iu, nm = load_ipu(self.log, args.ipu)
221
+ setattr(args, "ipu_iu", iu)
222
+ setattr(args, "ipu_nm", nm)
223
+
218
224
  if not self.args.no_ses:
219
225
  self.setup_session_db()
220
226
 
copyparty/up2k.py CHANGED
@@ -20,7 +20,7 @@ from copy import deepcopy
20
20
  from queue import Queue
21
21
 
22
22
  from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E
23
- from .authsrv import LEELOO_DALLAS, SSEELOG, VFS, AuthSrv
23
+ from .authsrv import LEELOO_DALLAS, SEESLOG, VFS, AuthSrv
24
24
  from .bos import bos
25
25
  from .cfg import vf_bmap, vf_cmap, vf_vmap
26
26
  from .fsutil import Fstab
@@ -353,17 +353,18 @@ class Up2k(object):
353
353
  return '[{"timeout":1}]'
354
354
 
355
355
  ret = []
356
+ userset = set([(uname or "\n"), "*"])
356
357
  try:
357
358
  for ptop, tab2 in self.registry.items():
358
359
  cfg = self.flags.get(ptop, {}).get("u2abort", 1)
359
360
  if not cfg:
360
361
  continue
361
362
  addr = (ip or "\n") if cfg in (1, 2) else ""
362
- user = (uname or "\n") if cfg in (1, 3) else ""
363
+ user = userset if cfg in (1, 3) else None
363
364
  for job in tab2.values():
364
365
  if (
365
366
  "done" in job
366
- or (user and user != job["user"])
367
+ or (user and job["user"] not in user)
367
368
  or (addr and addr != job["addr"])
368
369
  ):
369
370
  continue
@@ -1008,6 +1009,7 @@ class Up2k(object):
1008
1009
  vpath = k
1009
1010
 
1010
1011
  _, flags = self._expr_idx_filter(flags)
1012
+ n4g = bool(flags.get("noforget"))
1011
1013
 
1012
1014
  ft = "\033[0;32m{}{:.0}"
1013
1015
  ff = "\033[0;35m{}{:.0}"
@@ -1065,21 +1067,35 @@ class Up2k(object):
1065
1067
  for job in reg2.values():
1066
1068
  job["dwrk"] = job["wark"]
1067
1069
 
1070
+ rm = []
1068
1071
  for k, job in reg2.items():
1069
1072
  job["ptop"] = ptop
1073
+ if "done" in job:
1074
+ job["need"] = job["hash"] = emptylist
1075
+ else:
1076
+ if "need" not in job:
1077
+ job["need"] = []
1078
+ if "hash" not in job:
1079
+ job["hash"] = []
1080
+
1070
1081
  fp = djoin(ptop, job["prel"], job["name"])
1071
1082
  if bos.path.exists(fp):
1072
1083
  reg[k] = job
1073
1084
  if "done" in job:
1074
- job["need"] = job["hash"] = emptylist
1075
1085
  continue
1076
1086
  job["poke"] = time.time()
1077
1087
  job["busy"] = {}
1078
1088
  else:
1079
1089
  self.log("ign deleted file in snap: [{}]".format(fp))
1090
+ if not n4g:
1091
+ rm.append(k)
1092
+ continue
1093
+
1094
+ for x in rm:
1095
+ del reg2[x]
1080
1096
 
1081
1097
  if drp is None:
1082
- drp = [k for k, v in reg.items() if not v.get("need", [])]
1098
+ drp = [k for k, v in reg.items() if not v["need"]]
1083
1099
  else:
1084
1100
  drp = [x for x in drp if x in reg]
1085
1101
 
@@ -2860,9 +2876,6 @@ class Up2k(object):
2860
2876
  "user": cj["user"],
2861
2877
  "addr": ip,
2862
2878
  "at": at,
2863
- "hash": [],
2864
- "need": [],
2865
- "busy": {},
2866
2879
  }
2867
2880
  for k in ["life"]:
2868
2881
  if k in cj:
@@ -2896,17 +2909,20 @@ class Up2k(object):
2896
2909
  hashes2, st = self._hashlist_from_file(orig_ap)
2897
2910
  wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
2898
2911
  if dwark != wark2:
2899
- t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s"
2900
- self.log(t % (wark2, dwark, orig_ap))
2912
+ t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s\n%s"
2913
+ self.log(t % (wark2, dwark, orig_ap, rj))
2901
2914
  lost.append(dupe[3:])
2902
2915
  continue
2903
2916
  data_ok = True
2904
2917
  job = rj
2905
2918
  break
2906
2919
 
2907
- if job and wark in reg:
2908
- # self.log("pop " + wark + " " + job["name"] + " handle_json db", 4)
2909
- del reg[wark]
2920
+ if job:
2921
+ if wark in reg:
2922
+ del reg[wark]
2923
+ job["hash"] = job["need"] = []
2924
+ job["done"] = True
2925
+ job["busy"] = {}
2910
2926
 
2911
2927
  if lost:
2912
2928
  c2 = None
@@ -2934,7 +2950,7 @@ class Up2k(object):
2934
2950
  path = djoin(rj["ptop"], rj["prel"], fn)
2935
2951
  try:
2936
2952
  st = bos.stat(path)
2937
- if st.st_size > 0 or not rj["need"]:
2953
+ if st.st_size > 0 or "done" in rj:
2938
2954
  # upload completed or both present
2939
2955
  break
2940
2956
  except:
@@ -2948,13 +2964,13 @@ class Up2k(object):
2948
2964
  inc_ap = djoin(cj["ptop"], cj["prel"], cj["name"])
2949
2965
  orig_ap = djoin(rj["ptop"], rj["prel"], rj["name"])
2950
2966
 
2951
- if self.args.nw or n4g or not st:
2967
+ if self.args.nw or n4g or not st or "done" not in rj:
2952
2968
  pass
2953
2969
 
2954
2970
  elif st.st_size != rj["size"]:
2955
- t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}"
2971
+ t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}\n{}"
2956
2972
  t = t.format(
2957
- wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path
2973
+ wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path, rj
2958
2974
  )
2959
2975
  self.log(t)
2960
2976
  del reg[wark]
@@ -2964,8 +2980,8 @@ class Up2k(object):
2964
2980
  hashes2, _ = self._hashlist_from_file(orig_ap)
2965
2981
  wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
2966
2982
  if wark != wark2:
2967
- t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s"
2968
- self.log(t % (wark2, wark, orig_ap))
2983
+ t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s\n%s"
2984
+ self.log(t % (wark2, wark, orig_ap, rj))
2969
2985
  del reg[wark]
2970
2986
 
2971
2987
  if job or wark in reg:
@@ -2980,7 +2996,7 @@ class Up2k(object):
2980
2996
  dst = djoin(cj["ptop"], cj["prel"], cj["name"])
2981
2997
  vsrc = djoin(job["vtop"], job["prel"], job["name"])
2982
2998
  vsrc = vsrc.replace("\\", "/") # just for prints anyways
2983
- if job["need"]:
2999
+ if "done" not in job:
2984
3000
  self.log("unfinished:\n {0}\n {1}".format(src, dst))
2985
3001
  err = "partial upload exists at a different location; please resume uploading here instead:\n"
2986
3002
  err += "/" + quotep(vsrc) + " "
@@ -3341,14 +3357,14 @@ class Up2k(object):
3341
3357
 
3342
3358
  def handle_chunks(
3343
3359
  self, ptop , wark , chashes
3344
- ) :
3360
+ ) :
3345
3361
  with self.mutex, self.reg_mutex:
3346
3362
  self.db_act = self.vol_act[ptop] = time.time()
3347
3363
  job = self.registry[ptop].get(wark)
3348
3364
  if not job:
3349
3365
  known = " ".join([x for x in self.registry[ptop].keys()])
3350
3366
  self.log("unknown wark [{}], known: {}".format(wark, known))
3351
- raise Pebkac(400, "unknown wark" + SSEELOG)
3367
+ raise Pebkac(400, "unknown wark" + SEESLOG)
3352
3368
 
3353
3369
  if "t0c" not in job:
3354
3370
  job["t0c"] = time.time()
@@ -3364,7 +3380,7 @@ class Up2k(object):
3364
3380
  try:
3365
3381
  nchunk = uniq.index(chashes[0])
3366
3382
  except:
3367
- raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0]))
3383
+ raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0],))
3368
3384
  expanded = [chashes[0]]
3369
3385
  for prefix in chashes[1:]:
3370
3386
  nchunk += 1
@@ -3398,7 +3414,7 @@ class Up2k(object):
3398
3414
  for chash in chashes:
3399
3415
  nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
3400
3416
  if not nchunk:
3401
- raise Pebkac(400, "unknown chunk %s" % (chash))
3417
+ raise Pebkac(400, "unknown chunk %s" % (chash,))
3402
3418
 
3403
3419
  ofs = [chunksize * x for x in nchunk]
3404
3420
  coffsets.append(ofs)
@@ -3423,7 +3439,7 @@ class Up2k(object):
3423
3439
 
3424
3440
  job["poke"] = time.time()
3425
3441
 
3426
- return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
3442
+ return chashes, chunksize, coffsets, path, job["lmod"], job["size"], job["sprs"]
3427
3443
 
3428
3444
  def fast_confirm_chunks(
3429
3445
  self, ptop , wark , chashes
@@ -3492,11 +3508,13 @@ class Up2k(object):
3492
3508
  src = djoin(pdir, job["tnam"])
3493
3509
  dst = djoin(pdir, job["name"])
3494
3510
  except Exception as ex:
3495
- raise Pebkac(500, "finish_upload, wark, " + repr(ex))
3511
+ self.log(min_ex(), 1)
3512
+ raise Pebkac(500, "finish_upload, wark, %r%s" % (ex, SEESLOG))
3496
3513
 
3497
3514
  if job["need"]:
3498
- t = "finish_upload {} with remaining chunks {}"
3499
- raise Pebkac(500, t.format(wark, job["need"]))
3515
+ self.log(min_ex(), 1)
3516
+ t = "finish_upload %s with remaining chunks %s%s"
3517
+ raise Pebkac(500, t % (wark, job["need"], SEESLOG))
3500
3518
 
3501
3519
  upt = job.get("at") or time.time()
3502
3520
  vflags = self.flags[ptop]
@@ -3812,10 +3830,12 @@ class Up2k(object):
3812
3830
  with self.mutex, self.reg_mutex:
3813
3831
  abrt_cfg = self.flags.get(ptop, {}).get("u2abort", 1)
3814
3832
  addr = (ip or "\n") if abrt_cfg in (1, 2) else ""
3815
- user = (uname or "\n") if abrt_cfg in (1, 3) else ""
3833
+ user = ((uname or "\n"), "*") if abrt_cfg in (1, 3) else None
3816
3834
  reg = self.registry.get(ptop, {}) if abrt_cfg else {}
3817
3835
  for wark, job in reg.items():
3818
- if (user and user != job["user"]) or (addr and addr != job["addr"]):
3836
+ if (addr and addr != job["addr"]) or (
3837
+ user and job["user"] not in user
3838
+ ):
3819
3839
  continue
3820
3840
  jrem = djoin(job["prel"], job["name"])
3821
3841
  if ANYWIN:
@@ -4015,7 +4035,9 @@ class Up2k(object):
4015
4035
  self.db_act = self.vol_act[dbv.realpath] = time.time()
4016
4036
  svpf = "/".join(x for x in [dbv.vpath, vrem, fn[0]] if x)
4017
4037
  if not svpf.startswith(svp + "/"): # assert
4018
- raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
4038
+ self.log(min_ex(), 1)
4039
+ t = "mv: bug at %s, top %s%s"
4040
+ raise Pebkac(500, t % (svpf, svp, SEESLOG))
4019
4041
 
4020
4042
  dvpf = dvp + svpf[len(svp) :]
4021
4043
  self._mv_file(uname, ip, svpf, dvpf, curs)
@@ -4030,7 +4052,9 @@ class Up2k(object):
4030
4052
  for zsl in (rm_ok, rm_ng):
4031
4053
  for ap in reversed(zsl):
4032
4054
  if not ap.startswith(sabs):
4033
- raise Pebkac(500, "mv_d: bug at {}, top {}".format(ap, sabs))
4055
+ self.log(min_ex(), 1)
4056
+ t = "mv_d: bug at %s, top %s%s"
4057
+ raise Pebkac(500, t % (ap, sabs, SEESLOG))
4034
4058
 
4035
4059
  rem = ap[len(sabs) :].replace(os.sep, "/").lstrip("/")
4036
4060
  vp = vjoin(dvp, rem)
copyparty/util.py CHANGED
@@ -643,11 +643,15 @@ class HLog(logging.Handler):
643
643
 
644
644
 
645
645
  class NetMap(object):
646
- def __init__(self, ips , cidrs , keep_lo=False) :
646
+ def __init__(
647
+ self, ips , cidrs , keep_lo=False, strict_cidr=False
648
+ ) :
647
649
  """
648
650
  ips: list of plain ipv4/ipv6 IPs, not cidr
649
651
  cidrs: list of cidr-notation IPs (ip/prefix)
650
652
  """
653
+ self.mutex = threading.Lock()
654
+
651
655
  if "::" in ips:
652
656
  ips = [x for x in ips if x != "::"] + list(
653
657
  [x.split("/")[0] for x in cidrs if ":" in x]
@@ -674,7 +678,7 @@ class NetMap(object):
674
678
  bip = socket.inet_pton(fam, ip.split("/")[0])
675
679
  self.bip.append(bip)
676
680
  self.b2sip[bip] = ip.split("/")[0]
677
- self.b2net[bip] = (IPv6Network if v6 else IPv4Network)(ip, False)
681
+ self.b2net[bip] = (IPv6Network if v6 else IPv4Network)(ip, strict_cidr)
678
682
 
679
683
  self.bip.sort(reverse=True)
680
684
 
@@ -685,8 +689,10 @@ class NetMap(object):
685
689
  try:
686
690
  return self.cache[ip]
687
691
  except:
688
- pass
692
+ with self.mutex:
693
+ return self._map(ip)
689
694
 
695
+ def _map(self, ip ) :
690
696
  v6 = ":" in ip
691
697
  ci = IPv6Address(ip) if v6 else IPv4Address(ip)
692
698
  bip = next((x for x in self.bip if ci in self.b2net[x]), None)
@@ -2592,6 +2598,31 @@ def build_netmap(csv ):
2592
2598
  return NetMap(ips, cidrs, True)
2593
2599
 
2594
2600
 
2601
+ def load_ipu(log , ipus ) :
2602
+ ip_u = {"": "*"}
2603
+ cidr_u = {}
2604
+ for ipu in ipus:
2605
+ try:
2606
+ cidr, uname = ipu.split("=")
2607
+ cip, csz = cidr.split("/")
2608
+ except:
2609
+ t = "\n invalid value %r for argument --ipu; must be CIDR=UNAME (192.168.0.0/16=amelia)"
2610
+ raise Exception(t % (ipu,))
2611
+ uname2 = cidr_u.get(cidr)
2612
+ if uname2 is not None:
2613
+ t = "\n invalid value %r for argument --ipu; cidr %s already mapped to %r"
2614
+ raise Exception(t % (ipu, cidr, uname2))
2615
+ cidr_u[cidr] = uname
2616
+ ip_u[cip] = uname
2617
+ try:
2618
+ nm = NetMap(["::"], list(cidr_u.keys()), True, True)
2619
+ except Exception as ex:
2620
+ t = "failed to translate --ipu into netmap, probably due to invalid config: %r"
2621
+ log("root", t % (ex,), 1)
2622
+ raise
2623
+ return ip_u, nm
2624
+
2625
+
2595
2626
  def yieldfile(fn , bufsz ) :
2596
2627
  readsz = min(bufsz, 128 * 1024)
2597
2628
  with open(fsenc(fn), "rb", bufsz) as f:
@@ -2606,10 +2637,12 @@ def yieldfile(fn , bufsz ) :
2606
2637
  def hashcopy(
2607
2638
  fin ,
2608
2639
  fout ,
2609
- slp = 0,
2610
- max_sz = 0,
2640
+ hashobj ,
2641
+ max_sz ,
2642
+ slp ,
2611
2643
  ) :
2612
- hashobj = hashlib.sha512()
2644
+ if not hashobj:
2645
+ hashobj = hashlib.sha512()
2613
2646
  tlen = 0
2614
2647
  for buf in fin:
2615
2648
  tlen += len(buf)
copyparty/web/a/u2c.py CHANGED
@@ -1,8 +1,8 @@
1
1
  #!/usr/bin/env python3
2
2
  from __future__ import print_function, unicode_literals
3
3
 
4
- S_VERSION = "2.1"
5
- S_BUILD_DT = "2024-09-23"
4
+ S_VERSION = "2.4"
5
+ S_BUILD_DT = "2024-10-16"
6
6
 
7
7
  """
8
8
  u2c.py: upload to copyparty
@@ -62,6 +62,9 @@ else:
62
62
 
63
63
  unicode = str
64
64
 
65
+
66
+ WTF8 = "replace" if PY2 else "surrogateescape"
67
+
65
68
  VT100 = platform.system() != "Windows"
66
69
 
67
70
 
@@ -228,7 +231,7 @@ class File(object):
228
231
  self.lmod = lmod # type: float
229
232
 
230
233
  self.abs = os.path.join(top, rel) # type: bytes
231
- self.name = self.rel.split(b"/")[-1].decode("utf-8", "replace") # type: str
234
+ self.name = self.rel.split(b"/")[-1].decode("utf-8", WTF8) # type: str
232
235
 
233
236
  # set by get_hashlist
234
237
  self.cids = [] # type: list[tuple[str, int, int]] # [ hash, ofs, sz ]
@@ -267,10 +270,41 @@ class FileSlice(object):
267
270
  raise Exception(9)
268
271
  tlen += clen
269
272
 
270
- self.len = tlen
273
+ self.len = self.tlen = tlen
271
274
  self.cdr = self.car + self.len
272
275
  self.ofs = 0 # type: int
273
- self.f = open(file.abs, "rb", 512 * 1024)
276
+
277
+ self.f = None
278
+ self.seek = self._seek0
279
+ self.read = self._read0
280
+
281
+ def subchunk(self, maxsz, nth):
282
+ if self.tlen <= maxsz:
283
+ return -1
284
+
285
+ if not nth:
286
+ self.car0 = self.car
287
+ self.cdr0 = self.cdr
288
+
289
+ self.car = self.car0 + maxsz * nth
290
+ if self.car >= self.cdr0:
291
+ return -2
292
+
293
+ self.cdr = self.car + min(self.cdr0 - self.car, maxsz)
294
+ self.len = self.cdr - self.car
295
+ self.seek(0)
296
+ return nth
297
+
298
+ def unsub(self):
299
+ self.car = self.car0
300
+ self.cdr = self.cdr0
301
+ self.len = self.tlen
302
+
303
+ def _open(self):
304
+ self.seek = self._seek
305
+ self.read = self._read
306
+
307
+ self.f = open(self.file.abs, "rb", 512 * 1024)
274
308
  self.f.seek(self.car)
275
309
 
276
310
  # https://stackoverflow.com/questions/4359495/what-is-exactly-a-file-like-object-in-python
@@ -282,10 +316,14 @@ class FileSlice(object):
282
316
  except:
283
317
  pass # py27 probably
284
318
 
319
+ def close(self, *a, **ka):
320
+ return # until _open
321
+
285
322
  def tell(self):
286
323
  return self.ofs
287
324
 
288
- def seek(self, ofs, wh=0):
325
+ def _seek(self, ofs, wh=0):
326
+
289
327
  if wh == 1:
290
328
  ofs = self.ofs + ofs
291
329
  elif wh == 2:
@@ -299,12 +337,21 @@ class FileSlice(object):
299
337
  self.ofs = ofs
300
338
  self.f.seek(self.car + ofs)
301
339
 
302
- def read(self, sz):
340
+ def _read(self, sz):
341
+
303
342
  sz = min(sz, self.len - self.ofs)
304
343
  ret = self.f.read(sz)
305
344
  self.ofs += len(ret)
306
345
  return ret
307
346
 
347
+ def _seek0(self, ofs, wh=0):
348
+ self._open()
349
+ return self.seek(ofs, wh)
350
+
351
+ def _read0(self, sz):
352
+ self._open()
353
+ return self.read(sz)
354
+
308
355
 
309
356
  class MTHash(object):
310
357
  def __init__(self, cores):
@@ -557,13 +604,17 @@ def walkdir(err, top, excl, seen):
557
604
  for ap, inf in sorted(statdir(err, top)):
558
605
  if excl.match(ap):
559
606
  continue
560
- yield ap, inf
561
607
  if stat.S_ISDIR(inf.st_mode):
608
+ yield ap, inf
562
609
  try:
563
610
  for x in walkdir(err, ap, excl, seen):
564
611
  yield x
565
612
  except Exception as ex:
566
613
  err.append((ap, str(ex)))
614
+ elif stat.S_ISREG(inf.st_mode):
615
+ yield ap, inf
616
+ else:
617
+ err.append((ap, "irregular filetype 0%o" % (inf.st_mode,)))
567
618
 
568
619
 
569
620
  def walkdirs(err, tops, excl):
@@ -609,11 +660,12 @@ def walkdirs(err, tops, excl):
609
660
 
610
661
  # mostly from copyparty/util.py
611
662
  def quotep(btxt):
663
+ # type: (bytes) -> bytes
612
664
  quot1 = quote(btxt, safe=b"/")
613
665
  if not PY2:
614
666
  quot1 = quot1.encode("ascii")
615
667
 
616
- return quot1.replace(b" ", b"+") # type: ignore
668
+ return quot1.replace(b" ", b"%20") # type: ignore
617
669
 
618
670
 
619
671
  # from copyparty/util.py
@@ -641,7 +693,7 @@ def up2k_chunksize(filesize):
641
693
  while True:
642
694
  for mul in [1, 2]:
643
695
  nchunks = math.ceil(filesize * 1.0 / chunksize)
644
- if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks < 4096):
696
+ if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
645
697
  return chunksize
646
698
 
647
699
  chunksize += stepsize
@@ -720,7 +772,7 @@ def handshake(ar, file, search):
720
772
  url = file.url
721
773
  else:
722
774
  if b"/" in file.rel:
723
- url = quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8", "replace")
775
+ url = quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8")
724
776
  else:
725
777
  url = ""
726
778
  url = ar.vtop + url
@@ -728,6 +780,7 @@ def handshake(ar, file, search):
728
780
  while True:
729
781
  sc = 600
730
782
  txt = ""
783
+ t0 = time.time()
731
784
  try:
732
785
  zs = json.dumps(req, separators=(",\n", ": "))
733
786
  sc, txt = web.req("POST", url, {}, zs.encode("utf-8"), MJ)
@@ -752,7 +805,9 @@ def handshake(ar, file, search):
752
805
  print("\nERROR: login required, or wrong password:\n%s" % (txt,))
753
806
  raise BadAuth()
754
807
 
755
- eprint("handshake failed, retrying: %s\n %s\n\n" % (file.name, em))
808
+ t = "handshake failed, retrying: %s\n t0=%.3f t1=%.3f td=%.3f\n %s\n\n"
809
+ now = time.time()
810
+ eprint(t % (file.name, t0, now, now - t0, em))
756
811
  time.sleep(ar.cd)
757
812
 
758
813
  try:
@@ -763,15 +818,15 @@ def handshake(ar, file, search):
763
818
  if search:
764
819
  return r["hits"], False
765
820
 
766
- file.url = r["purl"]
821
+ file.url = quotep(r["purl"].encode("utf-8", WTF8)).decode("utf-8")
767
822
  file.name = r["name"]
768
823
  file.wark = r["wark"]
769
824
 
770
825
  return r["hash"], r["sprs"]
771
826
 
772
827
 
773
- def upload(fsl, stats):
774
- # type: (FileSlice, str) -> None
828
+ def upload(fsl, stats, maxsz):
829
+ # type: (FileSlice, str, int) -> None
775
830
  """upload a range of file data, defined by one or more `cid` (chunk-hash)"""
776
831
 
777
832
  ctxt = fsl.cids[0]
@@ -789,21 +844,33 @@ def upload(fsl, stats):
789
844
  if stats:
790
845
  headers["X-Up2k-Stat"] = stats
791
846
 
847
+ nsub = 0
792
848
  try:
793
- sc, txt = web.req("POST", fsl.file.url, headers, fsl, MO)
794
-
795
- if sc == 400:
796
- if (
797
- "already being written" in txt
798
- or "already got that" in txt
799
- or "only sibling chunks" in txt
800
- ):
801
- fsl.file.nojoin = 1
802
-
803
- if sc >= 400:
804
- raise Exception("http %s: %s" % (sc, txt))
849
+ while nsub != -1:
850
+ nsub = fsl.subchunk(maxsz, nsub)
851
+ if nsub == -2:
852
+ return
853
+ if nsub >= 0:
854
+ headers["X-Up2k-Subc"] = str(maxsz * nsub)
855
+ headers.pop(CLEN, None)
856
+ nsub += 1
857
+
858
+ sc, txt = web.req("POST", fsl.file.url, headers, fsl, MO)
859
+
860
+ if sc == 400:
861
+ if (
862
+ "already being written" in txt
863
+ or "already got that" in txt
864
+ or "only sibling chunks" in txt
865
+ ):
866
+ fsl.file.nojoin = 1
867
+
868
+ if sc >= 400:
869
+ raise Exception("http %s: %s" % (sc, txt))
805
870
  finally:
806
871
  fsl.f.close()
872
+ if nsub != -1:
873
+ fsl.unsub()
807
874
 
808
875
 
809
876
  class Ctl(object):
@@ -869,8 +936,8 @@ class Ctl(object):
869
936
  self.hash_b = 0
870
937
  self.up_f = 0
871
938
  self.up_c = 0
872
- self.up_b = 0
873
- self.up_br = 0
939
+ self.up_b = 0 # num bytes handled
940
+ self.up_br = 0 # num bytes actually transferred
874
941
  self.uploader_busy = 0
875
942
  self.serialized = False
876
943
 
@@ -935,7 +1002,7 @@ class Ctl(object):
935
1002
  print(" %d up %s" % (ncs - nc, cid))
936
1003
  stats = "%d/0/0/%d" % (nf, self.nfiles - nf)
937
1004
  fslice = FileSlice(file, [cid])
938
- upload(fslice, stats)
1005
+ upload(fslice, stats, self.ar.szm)
939
1006
 
940
1007
  print(" ok!")
941
1008
  if file.recheck:
@@ -1013,11 +1080,14 @@ class Ctl(object):
1013
1080
  t = "%s eta @ %s/s, %s, %d# left\033[K" % (self.eta, spd, sleft, nleft)
1014
1081
  eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
1015
1082
 
1083
+ if self.ar.wlist:
1084
+ self.at_hash = time.time() - self.t0
1085
+
1016
1086
  if self.hash_b and self.at_hash:
1017
1087
  spd = humansize(self.hash_b / self.at_hash)
1018
1088
  eprint("\nhasher: %.2f sec, %s/s\n" % (self.at_hash, spd))
1019
- if self.up_b and self.at_up:
1020
- spd = humansize(self.up_b / self.at_up)
1089
+ if self.up_br and self.at_up:
1090
+ spd = humansize(self.up_br / self.at_up)
1021
1091
  eprint("upload: %.2f sec, %s/s\n" % (self.at_up, spd))
1022
1092
 
1023
1093
  if not self.recheck:
@@ -1051,7 +1121,7 @@ class Ctl(object):
1051
1121
  print(" ls ~{0}".format(srd))
1052
1122
  zt = (
1053
1123
  self.ar.vtop,
1054
- quotep(rd.replace(b"\\", b"/")).decode("utf-8", "replace"),
1124
+ quotep(rd.replace(b"\\", b"/")).decode("utf-8"),
1055
1125
  )
1056
1126
  sc, txt = web.req("GET", "%s%s?ls&lt&dots" % zt, {})
1057
1127
  if sc >= 400:
@@ -1060,7 +1130,7 @@ class Ctl(object):
1060
1130
  j = json.loads(txt)
1061
1131
  for f in j["dirs"] + j["files"]:
1062
1132
  rfn = f["href"].split("?")[0].rstrip("/")
1063
- ls[unquote(rfn.encode("utf-8", "replace"))] = f
1133
+ ls[unquote(rfn.encode("utf-8", WTF8))] = f
1064
1134
  except Exception as ex:
1065
1135
  print(" mkdir ~{0} ({1})".format(srd, ex))
1066
1136
 
@@ -1074,7 +1144,7 @@ class Ctl(object):
1074
1144
  lnodes = [x.split(b"/")[-1] for x in zls]
1075
1145
  bnames = [x for x in ls if x not in lnodes and x != b".hist"]
1076
1146
  vpath = self.ar.url.split("://")[-1].split("/", 1)[-1]
1077
- names = [x.decode("utf-8", "replace") for x in bnames]
1147
+ names = [x.decode("utf-8", WTF8) for x in bnames]
1078
1148
  locs = [vpath + srd + "/" + x for x in names]
1079
1149
  while locs:
1080
1150
  req = locs
@@ -1136,10 +1206,16 @@ class Ctl(object):
1136
1206
  self.up_b = self.hash_b
1137
1207
 
1138
1208
  if self.ar.wlist:
1209
+ vp = file.rel.decode("utf-8")
1210
+ if self.ar.chs:
1211
+ zsl = [
1212
+ "%s %d %d" % (zsii[0], n, zsii[1])
1213
+ for n, zsii in enumerate(file.cids)
1214
+ ]
1215
+ print("chs: %s\n%s" % (vp, "\n".join(zsl)))
1139
1216
  zsl = [self.ar.wsalt, str(file.size)] + [x[0] for x in file.kchunks]
1140
1217
  zb = hashlib.sha512("\n".join(zsl).encode("utf-8")).digest()[:33]
1141
1218
  wark = ub64enc(zb).decode("utf-8")
1142
- vp = file.rel.decode("utf-8")
1143
1219
  if self.ar.jw:
1144
1220
  print("%s %s" % (wark, vp))
1145
1221
  else:
@@ -1177,6 +1253,7 @@ class Ctl(object):
1177
1253
  self.q_upload.put(None)
1178
1254
  return
1179
1255
 
1256
+ chunksz = up2k_chunksize(file.size)
1180
1257
  upath = file.abs.decode("utf-8", "replace")
1181
1258
  if not VT100:
1182
1259
  upath = upath.lstrip("\\?")
@@ -1236,9 +1313,14 @@ class Ctl(object):
1236
1313
  file.up_c -= len(hs)
1237
1314
  for cid in hs:
1238
1315
  sz = file.kchunks[cid][1]
1316
+ self.up_br -= sz
1239
1317
  self.up_b -= sz
1240
1318
  file.up_b -= sz
1241
1319
 
1320
+ if hs and not file.up_b:
1321
+ # first hs of this file; is this an upload resume?
1322
+ file.up_b = chunksz * max(0, len(file.kchunks) - len(hs))
1323
+
1242
1324
  file.ucids = hs
1243
1325
 
1244
1326
  if not hs:
@@ -1252,7 +1334,7 @@ class Ctl(object):
1252
1334
  c1 = c2 = ""
1253
1335
 
1254
1336
  spd_h = humansize(file.size / file.t_hash, True)
1255
- if file.up_b:
1337
+ if file.up_c:
1256
1338
  t_up = file.t1_up - file.t0_up
1257
1339
  spd_u = humansize(file.size / t_up, True)
1258
1340
 
@@ -1262,14 +1344,13 @@ class Ctl(object):
1262
1344
  t = " found %s %s(%.2fs,%s/s)%s"
1263
1345
  print(t % (upath, c1, file.t_hash, spd_h, c2))
1264
1346
  else:
1265
- kw = "uploaded" if file.up_b else " found"
1347
+ kw = "uploaded" if file.up_c else " found"
1266
1348
  print("{0} {1}".format(kw, upath))
1267
1349
 
1268
1350
  self._check_if_done()
1269
1351
  continue
1270
1352
 
1271
- chunksz = up2k_chunksize(file.size)
1272
- njoin = (self.ar.sz * 1024 * 1024) // chunksz
1353
+ njoin = self.ar.sz // chunksz
1273
1354
  cs = hs[:]
1274
1355
  while cs:
1275
1356
  fsl = FileSlice(file, cs[:1])
@@ -1321,7 +1402,7 @@ class Ctl(object):
1321
1402
  )
1322
1403
 
1323
1404
  try:
1324
- upload(fsl, stats)
1405
+ upload(fsl, stats, self.ar.szm)
1325
1406
  except Exception as ex:
1326
1407
  t = "upload failed, retrying: %s #%s+%d (%s)\n"
1327
1408
  eprint(t % (file.name, cids[0][:8], len(cids) - 1, ex))
@@ -1365,7 +1446,7 @@ def main():
1365
1446
  cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
1366
1447
  hcores = min(cores, 3) # 4% faster than 4+ on py3.9 @ r5-4500U
1367
1448
 
1368
- ver = "{0} v{1} https://youtu.be/BIcOO6TLKaY".format(S_BUILD_DT, S_VERSION)
1449
+ ver = "{0}, v{1}".format(S_BUILD_DT, S_VERSION)
1369
1450
  if "--version" in sys.argv:
1370
1451
  print(ver)
1371
1452
  return
@@ -1403,12 +1484,14 @@ source file/folder selection uses rsync syntax, meaning that:
1403
1484
 
1404
1485
  ap = app.add_argument_group("file-ID calculator; enable with url '-' to list warks (file identifiers) instead of upload/search")
1405
1486
  ap.add_argument("--wsalt", type=unicode, metavar="S", default="hunter2", help="salt to use when creating warks; must match server config")
1487
+ ap.add_argument("--chs", action="store_true", help="verbose (print the hash/offset of each chunk in each file)")
1406
1488
  ap.add_argument("--jw", action="store_true", help="just identifier+filepath, not mtime/size too")
1407
1489
 
1408
1490
  ap = app.add_argument_group("performance tweaks")
1409
1491
  ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
1410
1492
  ap.add_argument("-J", type=int, metavar="CORES", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
1411
1493
  ap.add_argument("--sz", type=int, metavar="MiB", default=64, help="try to make each POST this big")
1494
+ ap.add_argument("--szm", type=int, metavar="MiB", default=96, help="max size of each POST (default is cloudflare max)")
1412
1495
  ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
1413
1496
  ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
1414
1497
  ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
@@ -1436,6 +1519,9 @@ source file/folder selection uses rsync syntax, meaning that:
1436
1519
  if ar.dr:
1437
1520
  ar.ow = True
1438
1521
 
1522
+ ar.sz *= 1024 * 1024
1523
+ ar.szm *= 1024 * 1024
1524
+
1439
1525
  ar.x = "|".join(ar.x or [])
1440
1526
 
1441
1527
  setattr(ar, "wlist", ar.url == "-")
Binary file
Binary file
copyparty/web/ui.css.gz CHANGED
Binary file
copyparty/web/up2k.js.gz CHANGED
Binary file
copyparty/web/util.js.gz CHANGED
Binary file
Binary file
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: copyparty
3
- Version: 1.15.6
3
+ Version: 1.15.8
4
4
  Summary: Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
5
5
  Author-email: ed <copyparty@ocv.me>
6
6
  License: MIT
@@ -134,6 +134,7 @@ turn almost any device into a file server with resumable uploads/downloads using
134
134
  * [event hooks](#event-hooks) - trigger a program on uploads, renames etc ([examples](./bin/hooks/))
135
135
  * [upload events](#upload-events) - the older, more powerful approach ([examples](./bin/mtag/))
136
136
  * [handlers](#handlers) - redefine behavior with plugins ([examples](./bin/handlers/))
137
+ * [ip auth](#ip-auth) - autologin based on IP range (CIDR)
137
138
  * [identity providers](#identity-providers) - replace copyparty passwords with oauth and such
138
139
  * [user-changeable passwords](#user-changeable-passwords) - if permitted, users can change their own passwords
139
140
  * [using the cloud as storage](#using-the-cloud-as-storage) - connecting to an aws s3 bucket and similar
@@ -272,7 +273,7 @@ also see [comparison to similar software](./docs/versus.md)
272
273
  * upload
273
274
  * ☑ basic: plain multipart, ie6 support
274
275
  * ☑ [up2k](#uploading): js, resumable, multithreaded
275
- * **no filesize limit!** ...unless you use Cloudflare, then it's 383.9 GiB
276
+ * **no filesize limit!** even on Cloudflare
276
277
  * ☑ stash: simple PUT filedropper
277
278
  * ☑ filename randomizer
278
279
  * ☑ write-only folders
@@ -707,7 +708,7 @@ up2k has several advantages:
707
708
  * uploads resume if you reboot your browser or pc, just upload the same files again
708
709
  * server detects any corruption; the client reuploads affected chunks
709
710
  * the client doesn't upload anything that already exists on the server
710
- * no filesize limit unless imposed by a proxy, for example Cloudflare, which blocks uploads over 383.9 GiB
711
+ * no filesize limit, even when a proxy limits the request size (for example Cloudflare)
711
712
  * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
712
713
  * the last-modified timestamp of the file is preserved
713
714
 
@@ -743,6 +744,8 @@ note that since up2k has to read each file twice, `[🎈] bup` can *theoreticall
743
744
 
744
745
  if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
745
746
 
747
+ if the server is behind a proxy which imposes a request-size limit, you can configure up2k to sneak below the limit with server-option `--u2sz` (the default is 96 MiB to support Cloudflare)
748
+
746
749
 
747
750
  ### file-search
748
751
 
@@ -1486,6 +1489,22 @@ redefine behavior with plugins ([examples](./bin/handlers/))
1486
1489
  replace 404 and 403 errors with something completely different (that's it for now)
1487
1490
 
1488
1491
 
1492
+ ## ip auth
1493
+
1494
+ autologin based on IP range (CIDR) , using the global-option `--ipu`
1495
+
1496
+ for example, if everyone with an IP that starts with `192.168.123` should automatically log in as the user `spartacus`, then you can either specify `--ipu=192.168.123.0/24=spartacus` as a commandline option, or put this in a config file:
1497
+
1498
+ ```yaml
1499
+ [global]
1500
+ ipu: 192.168.123.0/24=spartacus
1501
+ ```
1502
+
1503
+ repeat the option to map additional subnets
1504
+
1505
+ **be careful with this one!** if you have a reverseproxy, then you definitely want to make sure you have [real-ip](#real-ip) configured correctly, and it's probably a good idea to nullmap the reverseproxy's IP just in case; so if your reverseproxy is sending requests from `172.24.27.9` then that would be `--ipu=172.24.27.9/32=`
1506
+
1507
+
1489
1508
  ## identity providers
1490
1509
 
1491
1510
  replace copyparty passwords with oauth and such
@@ -1,7 +1,7 @@
1
1
  copyparty/__init__.py,sha256=Chqw7uXX4r_-a2p6-xthrrqVHFI4aZdW45sWU7UvqeE,2597
2
- copyparty/__main__.py,sha256=6yMejKWZGgZBsIJ3A_lkpieMdpd0q7-nUgbsFTrpcR4,109765
3
- copyparty/__version__.py,sha256=cN7xKCyFmKnXASldOC8xD6LcXlzSFiM6G6TpRgB0zh4,258
4
- copyparty/authsrv.py,sha256=clsE8whf32_eIRES1r7VvsVkc4OiJCjiBEJM6pYkwwI,98670
2
+ copyparty/__main__.py,sha256=oTzWyuZotTwi-dHpD41mVp_EW2mPSWDMKP8o_nxp9Yk,110180
3
+ copyparty/__version__.py,sha256=V6jLFRwMXDp4buHI89fYcQoINvqpP0XYLPVMn_QHEDg,258
4
+ copyparty/authsrv.py,sha256=-lImrFH6pm3gcI76vZiFfEgrQx3_STYTSS2soYX5y1Y,98711
5
5
  copyparty/broker_mp.py,sha256=jsHUM2BSfRVRyZT869iPCqYEHSqedk6VkwvygZwbEZE,4017
6
6
  copyparty/broker_mpw.py,sha256=PYFgQfssOCfdI6qayW1ZjO1j1-7oez094muhYMbPOz0,3339
7
7
  copyparty/broker_thr.py,sha256=MXrwjusP0z1LPURUhi5jx_TL3jrXhYcDrJPDSKu6EEU,1705
@@ -10,10 +10,10 @@ copyparty/cert.py,sha256=0ZAPeXeMR164vWn9GQU3JDKooYXEq_NOQkDeg543ivg,8009
10
10
  copyparty/cfg.py,sha256=33nLatBUmzRFKQ4KpoQei3ZY6EqRrlaHpQnvCNFXcHI,10112
11
11
  copyparty/dxml.py,sha256=lZpg-kn-kQsXRtNY1n6fRaS-b7uXzMCyv8ovKnhZcZc,1548
12
12
  copyparty/fsutil.py,sha256=5CshJWO7CflfaRRNOb3JxghUH7W5rmS_HWNmKfx42MM,4538
13
- copyparty/ftpd.py,sha256=VTx0gZUdK_0xRORMZT1XymSYvsXsCZs6-wsM2J4raYA,17459
14
- copyparty/httpcli.py,sha256=QWOSdYhOxU3i-uCeoUUCEYt2O6pXbi3KjXoTr0YZm4Y,191346
15
- copyparty/httpconn.py,sha256=DN09_r3cPFN4UGCVtAhnIH6cBIDKF-fe4IgMeoHUnSc,6809
16
- copyparty/httpsrv.py,sha256=TnfqA4edgO187y8fV0Q-lchnnD2bMSBTPZgncEjGMhY,17009
13
+ copyparty/ftpd.py,sha256=G_h1urfIikzfCWGXnW9p-rioWdNM_Je6vWYq0-QSbC8,17580
14
+ copyparty/httpcli.py,sha256=xhM8unCDyo7Gj3hmLdhncXUCgKHgb6a300bolO1WF4U,194152
15
+ copyparty/httpconn.py,sha256=mQSgljh0Q-jyWjF4tQLrHbRKRe9WKl19kGqsGMsJpWo,6880
16
+ copyparty/httpsrv.py,sha256=d_UiGnQKniBoEV68lNFgnYm-byda7uj56mFf-YC7piI,17223
17
17
  copyparty/ico.py,sha256=eWSxEae4wOCfheHl-m-wchYvFRAR_97kJDb4NGaB-Z8,3561
18
18
  copyparty/mdns.py,sha256=vC078llnL1v0pvL3mnwacuStFHPJUQuxo9Opj-IbHL4,18155
19
19
  copyparty/metrics.py,sha256=aV09nntEmKMIyde8xoPtj1ehDOQVQOHchRF4uMMNzqM,8855
@@ -24,15 +24,15 @@ copyparty/smbd.py,sha256=Or7RF13cl1r3ncnpVh8BqyAGqH2Oa04O9iPZWCoB0Bo,14609
24
24
  copyparty/ssdp.py,sha256=R1Z61GZOxBMF2Sk4RTxKWMOemogmcjEWG-CvLihd45k,7023
25
25
  copyparty/star.py,sha256=tV5BbX6AiQ7N4UU8DYtSTckNYeoeey4DBqq4LjfymbY,3818
26
26
  copyparty/sutil.py,sha256=JTMrQwcWH85hXB_cKG206eDZ967WZDGaP00AWvl_gB0,3214
27
- copyparty/svchub.py,sha256=qLG2CE8VHro7I6FnYMwiij0LcBC5lJw-hiucBag3NHY,39939
27
+ copyparty/svchub.py,sha256=FuQGFBm-lJfe28M-SMBLieHy8jdWIQMkONK2GcWZU_E,40105
28
28
  copyparty/szip.py,sha256=sDypi1_yR6-62fIZ_3D0L9PfIzCUiK_3JqcaJCvTBCs,8601
29
29
  copyparty/tcpsrv.py,sha256=l_vb9FoF0AJur0IoqHNUSBDqMgBO_MRUZeDszi1UNfY,19881
30
30
  copyparty/tftpd.py,sha256=jZbf2JpeJmkuQWJErmAPG-dKhtYNvIUHbkAgodSXw9Y,13582
31
31
  copyparty/th_cli.py,sha256=o6FMkerYvAXS455z3DUossVztu_nzFlYSQhs6qN6Jt8,4636
32
32
  copyparty/th_srv.py,sha256=hI9wY1E_9N9Cgqvtr8zADeVqqiLGTiTdAnYAA7WFvJw,29346
33
33
  copyparty/u2idx.py,sha256=JjgqwgJBNj6sTn4PJfuqM3VEHqlmoyGC5bk4_92K2h0,13414
34
- copyparty/up2k.py,sha256=UlycCjhC0C1tMwZvHXlkoXRTaFIEbivg-sXGEBQCBCs,164564
35
- copyparty/util.py,sha256=4VKiIzGPY6kb7PkTL3lfD1z-ewQKMQM5_J3meGirlS0,91338
34
+ copyparty/up2k.py,sha256=Zn33f8EkFpAnK_RTDyVNA97CKlS9MS3k5mnnvYh1w1k,165351
35
+ copyparty/util.py,sha256=7_-17F94TsLw644xLT8FfO_fH0DwViD54-grej3f8sY,92379
36
36
  copyparty/bos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
37
  copyparty/bos/bos.py,sha256=Wb7eWsXJgR5AFlBR9ZOyKrLTwy-Kct9RrGiOu4Jo37Y,1622
38
38
  copyparty/bos/path.py,sha256=yEjCq2ki9CvxA5sCT8pS0keEXwugs0ZeUyUhdBziOCI,777
@@ -54,10 +54,10 @@ copyparty/stolen/ifaddr/__init__.py,sha256=vpREjAyPubr5s1NJi91icXV3q1o4DrKAvHABw
54
54
  copyparty/stolen/ifaddr/_posix.py,sha256=-67NdfGrCktfQPakT2fLbjl2U00QMvyBGkSvrUuTOrU,2626
55
55
  copyparty/stolen/ifaddr/_shared.py,sha256=uNC4SdEIgdSLKvuUzsf1aM-H1Xrc_9mpLoOT43YukGs,6206
56
56
  copyparty/stolen/ifaddr/_win32.py,sha256=EE-QyoBgeB7lYQ6z62VjXNaRozaYfCkaJBHGNA8QtZM,4026
57
- copyparty/web/baguettebox.js.gz,sha256=4dS8-r4si84ca71l98672ahnRI86Aq95MU-bc5knykk,7962
57
+ copyparty/web/baguettebox.js.gz,sha256=YIaxFDsubJfGIdzzxA-cL6GwJVmpWZyaPhW9hHcOIIw,7964
58
58
  copyparty/web/browser.css.gz,sha256=4bAS9Xkl2fflhaxRSRSVoYQcpXsg1mCWxsYjId7phbU,11610
59
59
  copyparty/web/browser.html,sha256=ISpfvWEawufJCYZIqvuXiyUgiXgjmOTtScz4zrEaypI,4870
60
- copyparty/web/browser.js.gz,sha256=0WhsCM1Kr0PFryxz7hF31apLgvsJac4jWy8_ePfp_40,84989
60
+ copyparty/web/browser.js.gz,sha256=7rubbEoqFlNn7FPF0mz3L2LQeWuPzw95MYpIaZmcczE,84985
61
61
  copyparty/web/browser2.html,sha256=NRUZ08GH-e2YcGXcoz0UjYg6JIVF42u4IMX4HHwWTmg,1587
62
62
  copyparty/web/cf.html,sha256=lJThtNFNAQT1ClCHHlivAkDGE0LutedwopXD62Z8Nys,589
63
63
  copyparty/web/dbg-audio.js.gz,sha256=Ma-KZtK8LnmiwNvNKFKXMPYl_Nn_3U7GsJ6-DRWC2HE,688
@@ -79,13 +79,13 @@ copyparty/web/splash.html,sha256=pUbsso_W3Q7bso8fy8qxh-fHDrrLm39mBBTIlTeH63w,523
79
79
  copyparty/web/splash.js.gz,sha256=Xoccku-2vE3tABo-88q3Cl4koHs_AE76T8QvMy4u6T8,2540
80
80
  copyparty/web/svcs.html,sha256=P5YZimYLeQMT0uz6u3clQSNZRc5Zs0Ok-ffcbcGSYuc,11762
81
81
  copyparty/web/svcs.js.gz,sha256=k81ZvZ3I-f4fMHKrNGGOgOlvXnCBz0mVjD-8mieoWCA,520
82
- copyparty/web/ui.css.gz,sha256=dTAXEPgWB83CkYXlP4hWq18BrLFzRlYwxoCoM5rp-Yw,2786
83
- copyparty/web/up2k.js.gz,sha256=t6mUaIYN8lrEJoWO_-2MZ9nIuEplimQgCqdWSsHnuGA,22811
84
- copyparty/web/util.js.gz,sha256=uh_NAVPiMcOCTy9oxUiDKmKh1GokTKBPe3FYNMMUYlM,14781
85
- copyparty/web/w.hash.js.gz,sha256=7wP9EZQNXQxwZnCCFUVsi_-6TM9PLZJeZ9krutXRRj8,1060
82
+ copyparty/web/ui.css.gz,sha256=wloSacrHgP722hy4XiOvVY2GI9-V4zvfvzu84LLWS_o,2779
83
+ copyparty/web/up2k.js.gz,sha256=iMaZ2joij8ndkA2iFCTri6MnYRxXzQbRu9uwAnxZBj8,23096
84
+ copyparty/web/util.js.gz,sha256=NvjPYhIa0-C_NhUyW-Ra-XinUCRjj8G3pYq1zJHYWEk,14805
85
+ copyparty/web/w.hash.js.gz,sha256=5weFAxkcfHhu90tUKnVnTu04U_PQjhzUVRfvir199I0,1093
86
86
  copyparty/web/a/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
87
87
  copyparty/web/a/partyfuse.py,sha256=fa9bBYNJHvtWpNVjQyyFzx6JOK7MJL1u0zj80PBYQKs,27960
88
- copyparty/web/a/u2c.py,sha256=QmEZOPJfOVmtVjHWP7XATyzvAAY1OzelikPKdRShOLg,46740
88
+ copyparty/web/a/u2c.py,sha256=QA0t_k422uLRh8Kt2eQIpcGYaaUZA1CGz_S-z23Yk-4,49343
89
89
  copyparty/web/a/webdav-cfg.bat,sha256=Y4NoGZlksAIg4cBMb7KdJrpKC6Nx97onaTl6yMjaimk,1449
90
90
  copyparty/web/dd/2.png,sha256=gJ14XFPzaw95L6z92fSq9eMPikSQyu-03P1lgiGe0_I,258
91
91
  copyparty/web/dd/3.png,sha256=4lho8Koz5tV7jJ4ODo6GMTScZfkqsT05yp48EDFIlyg,252
@@ -106,9 +106,9 @@ copyparty/web/deps/prismd.css.gz,sha256=ObUlksQVr-OuYlTz-I4B23TeBg2QDVVGRnWBz8cV
106
106
  copyparty/web/deps/scp.woff2,sha256=w99BDU5i8MukkMEL-iW0YO9H4vFFZSPWxbkH70ytaAg,8612
107
107
  copyparty/web/deps/sha512.ac.js.gz,sha256=lFZaCLumgWxrvEuDr4bqdKHsqjX82AbVAb7_F45Yk88,7033
108
108
  copyparty/web/deps/sha512.hw.js.gz,sha256=vqoXeracj-99Z5MfY3jK2N4WiSzYQdfjy0RnUlQDhSU,8110
109
- copyparty-1.15.6.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
110
- copyparty-1.15.6.dist-info/METADATA,sha256=rFYHpfJWHe5_4nuyJ9BCvq4YxRoaDRIgdY-VBjjqLqs,137956
111
- copyparty-1.15.6.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
112
- copyparty-1.15.6.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
113
- copyparty-1.15.6.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
114
- copyparty-1.15.6.dist-info/RECORD,,
109
+ copyparty-1.15.8.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
110
+ copyparty-1.15.8.dist-info/METADATA,sha256=R5tW0crtzFZ4vKzGny4gQ6-IOERgBgj7LZtP14Huung,138915
111
+ copyparty-1.15.8.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
112
+ copyparty-1.15.8.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
113
+ copyparty-1.15.8.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
114
+ copyparty-1.15.8.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.1.0)
2
+ Generator: setuptools (75.2.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5