matrix-synapse 1.145.0rc2__cp310-abi3-manylinux_2_28_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of matrix-synapse might be problematic. Click here for more details.

Files changed (1636) hide show
  1. AUTHORS.rst +51 -0
  2. CHANGES.md +1732 -0
  3. CONTRIBUTING.md +3 -0
  4. Cargo.lock +1923 -0
  5. Cargo.toml +6 -0
  6. INSTALL.md +7 -0
  7. LICENSE-AGPL-3.0 +661 -0
  8. LICENSE-COMMERCIAL +6 -0
  9. UPGRADE.rst +7 -0
  10. book.toml +47 -0
  11. changelog.d/.gitignore +1 -0
  12. demo/.gitignore +4 -0
  13. demo/clean.sh +22 -0
  14. demo/start.sh +173 -0
  15. demo/stop.sh +14 -0
  16. docs/.sample_config_header.yaml +12 -0
  17. docs/CAPTCHA_SETUP.md +37 -0
  18. docs/README.md +86 -0
  19. docs/SUMMARY.md +126 -0
  20. docs/admin_api/README.rst +13 -0
  21. docs/admin_api/account_validity.md +47 -0
  22. docs/admin_api/client_server_api_extensions.md +67 -0
  23. docs/admin_api/event_reports.md +185 -0
  24. docs/admin_api/experimental_features.md +52 -0
  25. docs/admin_api/fetch_event.md +53 -0
  26. docs/admin_api/media_admin_api.md +372 -0
  27. docs/admin_api/purge_history_api.md +77 -0
  28. docs/admin_api/register_api.md +92 -0
  29. docs/admin_api/room_membership.md +35 -0
  30. docs/admin_api/rooms.md +1190 -0
  31. docs/admin_api/scheduled_tasks.md +54 -0
  32. docs/admin_api/server_notices.md +48 -0
  33. docs/admin_api/statistics.md +132 -0
  34. docs/admin_api/user_admin_api.md +1602 -0
  35. docs/admin_api/version_api.md +23 -0
  36. docs/ancient_architecture_notes.md +81 -0
  37. docs/application_services.md +36 -0
  38. docs/architecture.md +65 -0
  39. docs/auth_chain_diff.dot +32 -0
  40. docs/auth_chain_diff.dot.png +0 -0
  41. docs/auth_chain_difference_algorithm.md +141 -0
  42. docs/changelogs/CHANGES-2019.md +1039 -0
  43. docs/changelogs/CHANGES-2020.md +2145 -0
  44. docs/changelogs/CHANGES-2021.md +2573 -0
  45. docs/changelogs/CHANGES-2022.md +2766 -0
  46. docs/changelogs/CHANGES-2023.md +2202 -0
  47. docs/changelogs/CHANGES-2024.md +1586 -0
  48. docs/changelogs/CHANGES-pre-1.0.md +3641 -0
  49. docs/changelogs/README.md +1 -0
  50. docs/code_style.md +119 -0
  51. docs/consent_tracking.md +197 -0
  52. docs/delegate.md +111 -0
  53. docs/deprecation_policy.md +89 -0
  54. docs/development/cas.md +64 -0
  55. docs/development/contributing_guide.md +554 -0
  56. docs/development/database_schema.md +404 -0
  57. docs/development/demo.md +42 -0
  58. docs/development/dependencies.md +326 -0
  59. docs/development/experimental_features.md +37 -0
  60. docs/development/git.md +148 -0
  61. docs/development/img/git/branches.jpg +0 -0
  62. docs/development/img/git/clean.png +0 -0
  63. docs/development/img/git/squash.png +0 -0
  64. docs/development/internal_documentation/README.md +12 -0
  65. docs/development/internal_documentation/release_notes_review_checklist.md +12 -0
  66. docs/development/releases.md +37 -0
  67. docs/development/reviews.md +41 -0
  68. docs/development/room-dag-concepts.md +115 -0
  69. docs/development/saml.md +40 -0
  70. docs/development/synapse_architecture/cancellation.md +392 -0
  71. docs/development/synapse_architecture/faster_joins.md +375 -0
  72. docs/development/synapse_architecture/streams.md +198 -0
  73. docs/element_logo_white_bg.svg +94 -0
  74. docs/favicon.png +0 -0
  75. docs/favicon.svg +58 -0
  76. docs/federate.md +67 -0
  77. docs/jwt.md +105 -0
  78. docs/log_contexts.md +566 -0
  79. docs/manhole.md +101 -0
  80. docs/media_repository.md +78 -0
  81. docs/message_retention_policies.md +207 -0
  82. docs/metrics-howto.md +315 -0
  83. docs/modules/account_data_callbacks.md +106 -0
  84. docs/modules/account_validity_callbacks.md +57 -0
  85. docs/modules/add_extra_fields_to_client_events_unsigned.md +32 -0
  86. docs/modules/background_update_controller_callbacks.md +71 -0
  87. docs/modules/index.md +53 -0
  88. docs/modules/media_repository_callbacks.md +131 -0
  89. docs/modules/password_auth_provider_callbacks.md +270 -0
  90. docs/modules/porting_legacy_module.md +20 -0
  91. docs/modules/presence_router_callbacks.md +112 -0
  92. docs/modules/ratelimit_callbacks.md +43 -0
  93. docs/modules/spam_checker_callbacks.md +517 -0
  94. docs/modules/third_party_rules_callbacks.md +350 -0
  95. docs/modules/writing_a_module.md +141 -0
  96. docs/openid.md +783 -0
  97. docs/opentracing.md +99 -0
  98. docs/other/running_synapse_on_single_board_computers.md +75 -0
  99. docs/password_auth_providers.md +129 -0
  100. docs/postgres.md +269 -0
  101. docs/presence_router_module.md +247 -0
  102. docs/privacy_policy_templates/en/1.0.html +26 -0
  103. docs/privacy_policy_templates/en/success.html +11 -0
  104. docs/replication.md +36 -0
  105. docs/reverse_proxy.md +327 -0
  106. docs/room_and_user_statistics.md +22 -0
  107. docs/sample_config.yaml +47 -0
  108. docs/sample_log_config.yaml +75 -0
  109. docs/server_notices.md +66 -0
  110. docs/setup/forward_proxy.md +89 -0
  111. docs/setup/installation.md +656 -0
  112. docs/setup/security.md +41 -0
  113. docs/setup/turn/coturn.md +197 -0
  114. docs/setup/turn/eturnal.md +170 -0
  115. docs/spam_checker.md +122 -0
  116. docs/sso_mapping_providers.md +208 -0
  117. docs/structured_logging.md +80 -0
  118. docs/synctl_workers.md +36 -0
  119. docs/systemd-with-workers/README.md +101 -0
  120. docs/systemd-with-workers/system/matrix-synapse-worker@.service +26 -0
  121. docs/systemd-with-workers/system/matrix-synapse.service +23 -0
  122. docs/systemd-with-workers/system/matrix-synapse.target +6 -0
  123. docs/systemd-with-workers/workers/background_worker.yaml +4 -0
  124. docs/systemd-with-workers/workers/event_persister.yaml +20 -0
  125. docs/systemd-with-workers/workers/federation_sender.yaml +4 -0
  126. docs/systemd-with-workers/workers/generic_worker.yaml +11 -0
  127. docs/systemd-with-workers/workers/media_worker.yaml +11 -0
  128. docs/systemd-with-workers/workers/pusher_worker.yaml +4 -0
  129. docs/tcp_replication.md +258 -0
  130. docs/templates.md +254 -0
  131. docs/turn-howto.md +168 -0
  132. docs/upgrade.md +2876 -0
  133. docs/usage/administration/README.md +7 -0
  134. docs/usage/administration/admin_api/README.md +47 -0
  135. docs/usage/administration/admin_api/background_updates.md +109 -0
  136. docs/usage/administration/admin_api/federation.md +212 -0
  137. docs/usage/administration/admin_api/registration_tokens.md +298 -0
  138. docs/usage/administration/admin_faq.md +294 -0
  139. docs/usage/administration/backups.md +125 -0
  140. docs/usage/administration/database_maintenance_tools.md +18 -0
  141. docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md +77 -0
  142. docs/usage/administration/monthly_active_users.md +84 -0
  143. docs/usage/administration/request_log.md +44 -0
  144. docs/usage/administration/state_groups.md +25 -0
  145. docs/usage/administration/understanding_synapse_through_grafana_graphs.md +83 -0
  146. docs/usage/administration/useful_sql_for_admins.md +216 -0
  147. docs/usage/configuration/README.md +4 -0
  148. docs/usage/configuration/config_documentation.md +4768 -0
  149. docs/usage/configuration/homeserver_sample_config.md +17 -0
  150. docs/usage/configuration/logging_sample_config.md +19 -0
  151. docs/usage/configuration/user_authentication/README.md +15 -0
  152. docs/usage/configuration/user_authentication/refresh_tokens.md +139 -0
  153. docs/usage/configuration/user_authentication/single_sign_on/README.md +5 -0
  154. docs/usage/configuration/user_authentication/single_sign_on/cas.md +8 -0
  155. docs/usage/configuration/user_authentication/single_sign_on/saml.md +8 -0
  156. docs/user_directory.md +130 -0
  157. docs/website_files/README.md +35 -0
  158. docs/website_files/indent-section-headers.css +7 -0
  159. docs/website_files/remove-nav-buttons.css +8 -0
  160. docs/website_files/table-of-contents.css +47 -0
  161. docs/website_files/table-of-contents.js +148 -0
  162. docs/website_files/theme/index.hbs +324 -0
  163. docs/website_files/version-picker.css +78 -0
  164. docs/website_files/version-picker.js +147 -0
  165. docs/website_files/version.js +1 -0
  166. docs/welcome_and_overview.md +67 -0
  167. docs/workers.md +897 -0
  168. matrix_synapse-1.145.0rc2.dist-info/METADATA +260 -0
  169. matrix_synapse-1.145.0rc2.dist-info/RECORD +1636 -0
  170. matrix_synapse-1.145.0rc2.dist-info/WHEEL +5 -0
  171. matrix_synapse-1.145.0rc2.dist-info/entry_points.txt +13 -0
  172. matrix_synapse-1.145.0rc2.dist-info/licenses/AUTHORS.rst +51 -0
  173. matrix_synapse-1.145.0rc2.dist-info/licenses/LICENSE-AGPL-3.0 +661 -0
  174. matrix_synapse-1.145.0rc2.dist-info/licenses/LICENSE-COMMERCIAL +6 -0
  175. mypy.ini +115 -0
  176. rust/Cargo.toml +63 -0
  177. rust/build.rs +45 -0
  178. scripts-dev/build_debian_packages.py +228 -0
  179. scripts-dev/check-newsfragment.sh +62 -0
  180. scripts-dev/check_line_terminators.sh +29 -0
  181. scripts-dev/check_locked_deps_have_sdists.py +64 -0
  182. scripts-dev/check_schema_delta.py +240 -0
  183. scripts-dev/complement.sh +332 -0
  184. scripts-dev/config-lint.sh +15 -0
  185. scripts-dev/database-save.sh +15 -0
  186. scripts-dev/docker_update_debian_changelog.sh +70 -0
  187. scripts-dev/dump_macaroon.py +25 -0
  188. scripts-dev/federation_client.py +428 -0
  189. scripts-dev/gen_config_documentation.py +510 -0
  190. scripts-dev/generate_sample_config.sh +28 -0
  191. scripts-dev/lint.sh +141 -0
  192. scripts-dev/make_full_schema.sh +310 -0
  193. scripts-dev/mypy_synapse_plugin.py +918 -0
  194. scripts-dev/next_github_number.sh +9 -0
  195. scripts-dev/release.py +986 -0
  196. scripts-dev/schema_versions.py +182 -0
  197. scripts-dev/sign_json.py +172 -0
  198. synapse/__init__.py +97 -0
  199. synapse/_scripts/__init__.py +0 -0
  200. synapse/_scripts/export_signing_key.py +109 -0
  201. synapse/_scripts/generate_config.py +83 -0
  202. synapse/_scripts/generate_log_config.py +56 -0
  203. synapse/_scripts/generate_signing_key.py +55 -0
  204. synapse/_scripts/generate_workers_map.py +318 -0
  205. synapse/_scripts/hash_password.py +95 -0
  206. synapse/_scripts/move_remote_media_to_new_store.py +128 -0
  207. synapse/_scripts/register_new_matrix_user.py +402 -0
  208. synapse/_scripts/review_recent_signups.py +212 -0
  209. synapse/_scripts/synapse_port_db.py +1604 -0
  210. synapse/_scripts/synctl.py +365 -0
  211. synapse/_scripts/update_synapse_database.py +130 -0
  212. synapse/api/__init__.py +20 -0
  213. synapse/api/auth/__init__.py +207 -0
  214. synapse/api/auth/base.py +406 -0
  215. synapse/api/auth/internal.py +299 -0
  216. synapse/api/auth/mas.py +436 -0
  217. synapse/api/auth/msc3861_delegated.py +617 -0
  218. synapse/api/auth_blocking.py +144 -0
  219. synapse/api/constants.py +379 -0
  220. synapse/api/errors.py +913 -0
  221. synapse/api/filtering.py +537 -0
  222. synapse/api/presence.py +102 -0
  223. synapse/api/ratelimiting.py +481 -0
  224. synapse/api/room_versions.py +535 -0
  225. synapse/api/urls.py +118 -0
  226. synapse/app/__init__.py +62 -0
  227. synapse/app/_base.py +906 -0
  228. synapse/app/admin_cmd.py +388 -0
  229. synapse/app/appservice.py +30 -0
  230. synapse/app/client_reader.py +30 -0
  231. synapse/app/complement_fork_starter.py +206 -0
  232. synapse/app/event_creator.py +29 -0
  233. synapse/app/federation_reader.py +30 -0
  234. synapse/app/federation_sender.py +30 -0
  235. synapse/app/frontend_proxy.py +30 -0
  236. synapse/app/generic_worker.py +475 -0
  237. synapse/app/homeserver.py +505 -0
  238. synapse/app/media_repository.py +30 -0
  239. synapse/app/phone_stats_home.py +292 -0
  240. synapse/app/pusher.py +30 -0
  241. synapse/app/synchrotron.py +30 -0
  242. synapse/app/user_dir.py +31 -0
  243. synapse/appservice/__init__.py +458 -0
  244. synapse/appservice/api.py +567 -0
  245. synapse/appservice/scheduler.py +565 -0
  246. synapse/config/__init__.py +27 -0
  247. synapse/config/__main__.py +62 -0
  248. synapse/config/_base.py +1154 -0
  249. synapse/config/_base.pyi +216 -0
  250. synapse/config/_util.py +99 -0
  251. synapse/config/account_validity.py +116 -0
  252. synapse/config/api.py +141 -0
  253. synapse/config/appservice.py +210 -0
  254. synapse/config/auth.py +80 -0
  255. synapse/config/auto_accept_invites.py +43 -0
  256. synapse/config/background_updates.py +44 -0
  257. synapse/config/cache.py +231 -0
  258. synapse/config/captcha.py +90 -0
  259. synapse/config/cas.py +116 -0
  260. synapse/config/consent.py +73 -0
  261. synapse/config/database.py +184 -0
  262. synapse/config/emailconfig.py +367 -0
  263. synapse/config/experimental.py +601 -0
  264. synapse/config/federation.py +114 -0
  265. synapse/config/homeserver.py +141 -0
  266. synapse/config/jwt.py +55 -0
  267. synapse/config/key.py +447 -0
  268. synapse/config/logger.py +390 -0
  269. synapse/config/mas.py +192 -0
  270. synapse/config/matrixrtc.py +66 -0
  271. synapse/config/metrics.py +93 -0
  272. synapse/config/modules.py +40 -0
  273. synapse/config/oembed.py +185 -0
  274. synapse/config/oidc.py +509 -0
  275. synapse/config/password_auth_providers.py +82 -0
  276. synapse/config/push.py +64 -0
  277. synapse/config/ratelimiting.py +260 -0
  278. synapse/config/redis.py +74 -0
  279. synapse/config/registration.py +296 -0
  280. synapse/config/repository.py +311 -0
  281. synapse/config/retention.py +162 -0
  282. synapse/config/room.py +88 -0
  283. synapse/config/room_directory.py +165 -0
  284. synapse/config/saml2.py +251 -0
  285. synapse/config/server.py +1139 -0
  286. synapse/config/server_notices.py +84 -0
  287. synapse/config/spam_checker.py +66 -0
  288. synapse/config/sso.py +121 -0
  289. synapse/config/stats.py +54 -0
  290. synapse/config/third_party_event_rules.py +40 -0
  291. synapse/config/tls.py +192 -0
  292. synapse/config/tracer.py +71 -0
  293. synapse/config/user_directory.py +47 -0
  294. synapse/config/user_types.py +42 -0
  295. synapse/config/voip.py +59 -0
  296. synapse/config/workers.py +642 -0
  297. synapse/crypto/__init__.py +20 -0
  298. synapse/crypto/context_factory.py +278 -0
  299. synapse/crypto/event_signing.py +194 -0
  300. synapse/crypto/keyring.py +980 -0
  301. synapse/event_auth.py +1266 -0
  302. synapse/events/__init__.py +667 -0
  303. synapse/events/auto_accept_invites.py +216 -0
  304. synapse/events/builder.py +387 -0
  305. synapse/events/presence_router.py +243 -0
  306. synapse/events/snapshot.py +559 -0
  307. synapse/events/utils.py +924 -0
  308. synapse/events/validator.py +305 -0
  309. synapse/federation/__init__.py +22 -0
  310. synapse/federation/federation_base.py +382 -0
  311. synapse/federation/federation_client.py +2133 -0
  312. synapse/federation/federation_server.py +1543 -0
  313. synapse/federation/persistence.py +70 -0
  314. synapse/federation/send_queue.py +532 -0
  315. synapse/federation/sender/__init__.py +1165 -0
  316. synapse/federation/sender/per_destination_queue.py +886 -0
  317. synapse/federation/sender/transaction_manager.py +210 -0
  318. synapse/federation/transport/__init__.py +28 -0
  319. synapse/federation/transport/client.py +1199 -0
  320. synapse/federation/transport/server/__init__.py +334 -0
  321. synapse/federation/transport/server/_base.py +429 -0
  322. synapse/federation/transport/server/federation.py +910 -0
  323. synapse/federation/units.py +133 -0
  324. synapse/handlers/__init__.py +20 -0
  325. synapse/handlers/account.py +162 -0
  326. synapse/handlers/account_data.py +360 -0
  327. synapse/handlers/account_validity.py +362 -0
  328. synapse/handlers/admin.py +615 -0
  329. synapse/handlers/appservice.py +989 -0
  330. synapse/handlers/auth.py +2482 -0
  331. synapse/handlers/cas.py +413 -0
  332. synapse/handlers/deactivate_account.py +363 -0
  333. synapse/handlers/delayed_events.py +607 -0
  334. synapse/handlers/device.py +1869 -0
  335. synapse/handlers/devicemessage.py +399 -0
  336. synapse/handlers/directory.py +545 -0
  337. synapse/handlers/e2e_keys.py +1835 -0
  338. synapse/handlers/e2e_room_keys.py +455 -0
  339. synapse/handlers/event_auth.py +390 -0
  340. synapse/handlers/events.py +203 -0
  341. synapse/handlers/federation.py +2042 -0
  342. synapse/handlers/federation_event.py +2420 -0
  343. synapse/handlers/identity.py +812 -0
  344. synapse/handlers/initial_sync.py +528 -0
  345. synapse/handlers/jwt.py +120 -0
  346. synapse/handlers/message.py +2357 -0
  347. synapse/handlers/oidc.py +1801 -0
  348. synapse/handlers/pagination.py +811 -0
  349. synapse/handlers/password_policy.py +102 -0
  350. synapse/handlers/presence.py +2634 -0
  351. synapse/handlers/profile.py +656 -0
  352. synapse/handlers/push_rules.py +164 -0
  353. synapse/handlers/read_marker.py +79 -0
  354. synapse/handlers/receipts.py +351 -0
  355. synapse/handlers/register.py +1059 -0
  356. synapse/handlers/relations.py +623 -0
  357. synapse/handlers/reports.py +98 -0
  358. synapse/handlers/room.py +2449 -0
  359. synapse/handlers/room_list.py +632 -0
  360. synapse/handlers/room_member.py +2366 -0
  361. synapse/handlers/room_member_worker.py +146 -0
  362. synapse/handlers/room_policy.py +186 -0
  363. synapse/handlers/room_summary.py +1057 -0
  364. synapse/handlers/saml.py +524 -0
  365. synapse/handlers/search.py +723 -0
  366. synapse/handlers/send_email.py +209 -0
  367. synapse/handlers/set_password.py +71 -0
  368. synapse/handlers/sliding_sync/__init__.py +1961 -0
  369. synapse/handlers/sliding_sync/extensions.py +969 -0
  370. synapse/handlers/sliding_sync/room_lists.py +2317 -0
  371. synapse/handlers/sliding_sync/store.py +126 -0
  372. synapse/handlers/sso.py +1291 -0
  373. synapse/handlers/state_deltas.py +82 -0
  374. synapse/handlers/stats.py +322 -0
  375. synapse/handlers/sync.py +3096 -0
  376. synapse/handlers/thread_subscriptions.py +190 -0
  377. synapse/handlers/typing.py +612 -0
  378. synapse/handlers/ui_auth/__init__.py +48 -0
  379. synapse/handlers/ui_auth/checkers.py +332 -0
  380. synapse/handlers/user_directory.py +786 -0
  381. synapse/handlers/worker_lock.py +371 -0
  382. synapse/http/__init__.py +105 -0
  383. synapse/http/additional_resource.py +62 -0
  384. synapse/http/client.py +1377 -0
  385. synapse/http/connectproxyclient.py +316 -0
  386. synapse/http/federation/__init__.py +19 -0
  387. synapse/http/federation/matrix_federation_agent.py +490 -0
  388. synapse/http/federation/srv_resolver.py +196 -0
  389. synapse/http/federation/well_known_resolver.py +368 -0
  390. synapse/http/matrixfederationclient.py +1874 -0
  391. synapse/http/proxy.py +290 -0
  392. synapse/http/proxyagent.py +497 -0
  393. synapse/http/replicationagent.py +203 -0
  394. synapse/http/request_metrics.py +309 -0
  395. synapse/http/server.py +1111 -0
  396. synapse/http/servlet.py +1018 -0
  397. synapse/http/site.py +952 -0
  398. synapse/http/types.py +27 -0
  399. synapse/logging/__init__.py +31 -0
  400. synapse/logging/_remote.py +261 -0
  401. synapse/logging/_terse_json.py +95 -0
  402. synapse/logging/context.py +1209 -0
  403. synapse/logging/formatter.py +62 -0
  404. synapse/logging/handlers.py +99 -0
  405. synapse/logging/loggers.py +25 -0
  406. synapse/logging/opentracing.py +1132 -0
  407. synapse/logging/scopecontextmanager.py +160 -0
  408. synapse/media/_base.py +831 -0
  409. synapse/media/filepath.py +417 -0
  410. synapse/media/media_repository.py +1595 -0
  411. synapse/media/media_storage.py +703 -0
  412. synapse/media/oembed.py +277 -0
  413. synapse/media/preview_html.py +556 -0
  414. synapse/media/storage_provider.py +195 -0
  415. synapse/media/thumbnailer.py +833 -0
  416. synapse/media/url_previewer.py +884 -0
  417. synapse/metrics/__init__.py +748 -0
  418. synapse/metrics/_gc.py +219 -0
  419. synapse/metrics/_reactor_metrics.py +171 -0
  420. synapse/metrics/_types.py +38 -0
  421. synapse/metrics/background_process_metrics.py +555 -0
  422. synapse/metrics/common_usage_metrics.py +95 -0
  423. synapse/metrics/jemalloc.py +248 -0
  424. synapse/module_api/__init__.py +2131 -0
  425. synapse/module_api/callbacks/__init__.py +50 -0
  426. synapse/module_api/callbacks/account_validity_callbacks.py +106 -0
  427. synapse/module_api/callbacks/media_repository_callbacks.py +157 -0
  428. synapse/module_api/callbacks/ratelimit_callbacks.py +78 -0
  429. synapse/module_api/callbacks/spamchecker_callbacks.py +991 -0
  430. synapse/module_api/callbacks/third_party_event_rules_callbacks.py +592 -0
  431. synapse/module_api/errors.py +42 -0
  432. synapse/notifier.py +969 -0
  433. synapse/push/__init__.py +212 -0
  434. synapse/push/bulk_push_rule_evaluator.py +635 -0
  435. synapse/push/clientformat.py +126 -0
  436. synapse/push/emailpusher.py +334 -0
  437. synapse/push/httppusher.py +565 -0
  438. synapse/push/mailer.py +1009 -0
  439. synapse/push/presentable_names.py +216 -0
  440. synapse/push/push_tools.py +114 -0
  441. synapse/push/push_types.py +141 -0
  442. synapse/push/pusher.py +87 -0
  443. synapse/push/pusherpool.py +501 -0
  444. synapse/push/rulekinds.py +33 -0
  445. synapse/py.typed +0 -0
  446. synapse/replication/__init__.py +20 -0
  447. synapse/replication/http/__init__.py +68 -0
  448. synapse/replication/http/_base.py +469 -0
  449. synapse/replication/http/account_data.py +297 -0
  450. synapse/replication/http/deactivate_account.py +81 -0
  451. synapse/replication/http/delayed_events.py +62 -0
  452. synapse/replication/http/devices.py +254 -0
  453. synapse/replication/http/federation.py +334 -0
  454. synapse/replication/http/login.py +106 -0
  455. synapse/replication/http/membership.py +364 -0
  456. synapse/replication/http/presence.py +133 -0
  457. synapse/replication/http/push.py +156 -0
  458. synapse/replication/http/register.py +172 -0
  459. synapse/replication/http/send_events.py +182 -0
  460. synapse/replication/http/state.py +82 -0
  461. synapse/replication/http/streams.py +101 -0
  462. synapse/replication/tcp/__init__.py +56 -0
  463. synapse/replication/tcp/client.py +553 -0
  464. synapse/replication/tcp/commands.py +569 -0
  465. synapse/replication/tcp/context.py +41 -0
  466. synapse/replication/tcp/external_cache.py +156 -0
  467. synapse/replication/tcp/handler.py +922 -0
  468. synapse/replication/tcp/protocol.py +611 -0
  469. synapse/replication/tcp/redis.py +510 -0
  470. synapse/replication/tcp/resource.py +349 -0
  471. synapse/replication/tcp/streams/__init__.py +96 -0
  472. synapse/replication/tcp/streams/_base.py +765 -0
  473. synapse/replication/tcp/streams/events.py +287 -0
  474. synapse/replication/tcp/streams/federation.py +92 -0
  475. synapse/replication/tcp/streams/partial_state.py +80 -0
  476. synapse/res/providers.json +29 -0
  477. synapse/res/templates/_base.html +29 -0
  478. synapse/res/templates/account_previously_renewed.html +6 -0
  479. synapse/res/templates/account_renewed.html +6 -0
  480. synapse/res/templates/add_threepid.html +8 -0
  481. synapse/res/templates/add_threepid.txt +6 -0
  482. synapse/res/templates/add_threepid_failure.html +7 -0
  483. synapse/res/templates/add_threepid_success.html +6 -0
  484. synapse/res/templates/already_in_use.html +12 -0
  485. synapse/res/templates/already_in_use.txt +10 -0
  486. synapse/res/templates/auth_success.html +21 -0
  487. synapse/res/templates/invalid_token.html +6 -0
  488. synapse/res/templates/mail-Element.css +7 -0
  489. synapse/res/templates/mail-Vector.css +7 -0
  490. synapse/res/templates/mail-expiry.css +4 -0
  491. synapse/res/templates/mail.css +156 -0
  492. synapse/res/templates/notice_expiry.html +46 -0
  493. synapse/res/templates/notice_expiry.txt +7 -0
  494. synapse/res/templates/notif.html +51 -0
  495. synapse/res/templates/notif.txt +22 -0
  496. synapse/res/templates/notif_mail.html +59 -0
  497. synapse/res/templates/notif_mail.txt +10 -0
  498. synapse/res/templates/password_reset.html +10 -0
  499. synapse/res/templates/password_reset.txt +7 -0
  500. synapse/res/templates/password_reset_confirmation.html +15 -0
  501. synapse/res/templates/password_reset_failure.html +7 -0
  502. synapse/res/templates/password_reset_success.html +6 -0
  503. synapse/res/templates/recaptcha.html +42 -0
  504. synapse/res/templates/registration.html +12 -0
  505. synapse/res/templates/registration.txt +10 -0
  506. synapse/res/templates/registration_failure.html +6 -0
  507. synapse/res/templates/registration_success.html +6 -0
  508. synapse/res/templates/registration_token.html +18 -0
  509. synapse/res/templates/room.html +33 -0
  510. synapse/res/templates/room.txt +9 -0
  511. synapse/res/templates/sso.css +129 -0
  512. synapse/res/templates/sso_account_deactivated.html +25 -0
  513. synapse/res/templates/sso_auth_account_details.html +186 -0
  514. synapse/res/templates/sso_auth_account_details.js +116 -0
  515. synapse/res/templates/sso_auth_bad_user.html +26 -0
  516. synapse/res/templates/sso_auth_confirm.html +27 -0
  517. synapse/res/templates/sso_auth_success.html +26 -0
  518. synapse/res/templates/sso_error.html +71 -0
  519. synapse/res/templates/sso_footer.html +19 -0
  520. synapse/res/templates/sso_login_idp_picker.html +60 -0
  521. synapse/res/templates/sso_new_user_consent.html +30 -0
  522. synapse/res/templates/sso_partial_profile.html +19 -0
  523. synapse/res/templates/sso_redirect_confirm.html +39 -0
  524. synapse/res/templates/style.css +33 -0
  525. synapse/res/templates/terms.html +27 -0
  526. synapse/rest/__init__.py +197 -0
  527. synapse/rest/admin/__init__.py +395 -0
  528. synapse/rest/admin/_base.py +72 -0
  529. synapse/rest/admin/background_updates.py +171 -0
  530. synapse/rest/admin/devices.py +221 -0
  531. synapse/rest/admin/event_reports.py +173 -0
  532. synapse/rest/admin/events.py +69 -0
  533. synapse/rest/admin/experimental_features.py +137 -0
  534. synapse/rest/admin/federation.py +243 -0
  535. synapse/rest/admin/media.py +540 -0
  536. synapse/rest/admin/registration_tokens.py +358 -0
  537. synapse/rest/admin/rooms.py +1092 -0
  538. synapse/rest/admin/scheduled_tasks.py +70 -0
  539. synapse/rest/admin/server_notice_servlet.py +132 -0
  540. synapse/rest/admin/statistics.py +132 -0
  541. synapse/rest/admin/username_available.py +58 -0
  542. synapse/rest/admin/users.py +1634 -0
  543. synapse/rest/client/__init__.py +20 -0
  544. synapse/rest/client/_base.py +113 -0
  545. synapse/rest/client/account.py +937 -0
  546. synapse/rest/client/account_data.py +319 -0
  547. synapse/rest/client/account_validity.py +103 -0
  548. synapse/rest/client/appservice_ping.py +125 -0
  549. synapse/rest/client/auth.py +218 -0
  550. synapse/rest/client/auth_metadata.py +122 -0
  551. synapse/rest/client/capabilities.py +121 -0
  552. synapse/rest/client/delayed_events.py +165 -0
  553. synapse/rest/client/devices.py +587 -0
  554. synapse/rest/client/directory.py +211 -0
  555. synapse/rest/client/events.py +116 -0
  556. synapse/rest/client/filter.py +112 -0
  557. synapse/rest/client/initial_sync.py +65 -0
  558. synapse/rest/client/keys.py +678 -0
  559. synapse/rest/client/knock.py +104 -0
  560. synapse/rest/client/login.py +750 -0
  561. synapse/rest/client/login_token_request.py +127 -0
  562. synapse/rest/client/logout.py +93 -0
  563. synapse/rest/client/matrixrtc.py +52 -0
  564. synapse/rest/client/media.py +307 -0
  565. synapse/rest/client/mutual_rooms.py +145 -0
  566. synapse/rest/client/notifications.py +137 -0
  567. synapse/rest/client/openid.py +109 -0
  568. synapse/rest/client/password_policy.py +69 -0
  569. synapse/rest/client/presence.py +131 -0
  570. synapse/rest/client/profile.py +291 -0
  571. synapse/rest/client/push_rule.py +331 -0
  572. synapse/rest/client/pusher.py +181 -0
  573. synapse/rest/client/read_marker.py +104 -0
  574. synapse/rest/client/receipts.py +165 -0
  575. synapse/rest/client/register.py +1072 -0
  576. synapse/rest/client/relations.py +138 -0
  577. synapse/rest/client/rendezvous.py +76 -0
  578. synapse/rest/client/reporting.py +207 -0
  579. synapse/rest/client/room.py +1763 -0
  580. synapse/rest/client/room_keys.py +426 -0
  581. synapse/rest/client/room_upgrade_rest_servlet.py +112 -0
  582. synapse/rest/client/sendtodevice.py +85 -0
  583. synapse/rest/client/sync.py +1131 -0
  584. synapse/rest/client/tags.py +129 -0
  585. synapse/rest/client/thirdparty.py +130 -0
  586. synapse/rest/client/thread_subscriptions.py +247 -0
  587. synapse/rest/client/tokenrefresh.py +52 -0
  588. synapse/rest/client/transactions.py +150 -0
  589. synapse/rest/client/user_directory.py +99 -0
  590. synapse/rest/client/versions.py +193 -0
  591. synapse/rest/client/voip.py +88 -0
  592. synapse/rest/consent/__init__.py +0 -0
  593. synapse/rest/consent/consent_resource.py +210 -0
  594. synapse/rest/health.py +38 -0
  595. synapse/rest/key/__init__.py +20 -0
  596. synapse/rest/key/v2/__init__.py +40 -0
  597. synapse/rest/key/v2/local_key_resource.py +125 -0
  598. synapse/rest/key/v2/remote_key_resource.py +302 -0
  599. synapse/rest/media/__init__.py +0 -0
  600. synapse/rest/media/config_resource.py +53 -0
  601. synapse/rest/media/create_resource.py +90 -0
  602. synapse/rest/media/download_resource.py +110 -0
  603. synapse/rest/media/media_repository_resource.py +113 -0
  604. synapse/rest/media/preview_url_resource.py +77 -0
  605. synapse/rest/media/thumbnail_resource.py +142 -0
  606. synapse/rest/media/upload_resource.py +187 -0
  607. synapse/rest/media/v1/__init__.py +39 -0
  608. synapse/rest/media/v1/_base.py +23 -0
  609. synapse/rest/media/v1/media_storage.py +23 -0
  610. synapse/rest/media/v1/storage_provider.py +23 -0
  611. synapse/rest/synapse/__init__.py +20 -0
  612. synapse/rest/synapse/client/__init__.py +93 -0
  613. synapse/rest/synapse/client/federation_whitelist.py +66 -0
  614. synapse/rest/synapse/client/jwks.py +77 -0
  615. synapse/rest/synapse/client/new_user_consent.py +115 -0
  616. synapse/rest/synapse/client/oidc/__init__.py +45 -0
  617. synapse/rest/synapse/client/oidc/backchannel_logout_resource.py +42 -0
  618. synapse/rest/synapse/client/oidc/callback_resource.py +48 -0
  619. synapse/rest/synapse/client/password_reset.py +129 -0
  620. synapse/rest/synapse/client/pick_idp.py +107 -0
  621. synapse/rest/synapse/client/pick_username.py +153 -0
  622. synapse/rest/synapse/client/rendezvous.py +58 -0
  623. synapse/rest/synapse/client/saml2/__init__.py +42 -0
  624. synapse/rest/synapse/client/saml2/metadata_resource.py +46 -0
  625. synapse/rest/synapse/client/saml2/response_resource.py +52 -0
  626. synapse/rest/synapse/client/sso_register.py +56 -0
  627. synapse/rest/synapse/client/unsubscribe.py +88 -0
  628. synapse/rest/synapse/mas/__init__.py +71 -0
  629. synapse/rest/synapse/mas/_base.py +55 -0
  630. synapse/rest/synapse/mas/devices.py +239 -0
  631. synapse/rest/synapse/mas/users.py +469 -0
  632. synapse/rest/well_known.py +148 -0
  633. synapse/server.py +1279 -0
  634. synapse/server_notices/__init__.py +0 -0
  635. synapse/server_notices/consent_server_notices.py +136 -0
  636. synapse/server_notices/resource_limits_server_notices.py +215 -0
  637. synapse/server_notices/server_notices_manager.py +388 -0
  638. synapse/server_notices/server_notices_sender.py +67 -0
  639. synapse/server_notices/worker_server_notices_sender.py +46 -0
  640. synapse/spam_checker_api/__init__.py +31 -0
  641. synapse/state/__init__.py +1023 -0
  642. synapse/state/v1.py +369 -0
  643. synapse/state/v2.py +985 -0
  644. synapse/static/client/login/index.html +47 -0
  645. synapse/static/client/login/js/jquery-3.4.1.min.js +2 -0
  646. synapse/static/client/login/js/login.js +291 -0
  647. synapse/static/client/login/spinner.gif +0 -0
  648. synapse/static/client/login/style.css +79 -0
  649. synapse/static/index.html +63 -0
  650. synapse/storage/__init__.py +43 -0
  651. synapse/storage/_base.py +245 -0
  652. synapse/storage/admin_client_config.py +25 -0
  653. synapse/storage/background_updates.py +1189 -0
  654. synapse/storage/controllers/__init__.py +57 -0
  655. synapse/storage/controllers/persist_events.py +1237 -0
  656. synapse/storage/controllers/purge_events.py +456 -0
  657. synapse/storage/controllers/state.py +950 -0
  658. synapse/storage/controllers/stats.py +119 -0
  659. synapse/storage/database.py +2720 -0
  660. synapse/storage/databases/__init__.py +175 -0
  661. synapse/storage/databases/main/__init__.py +420 -0
  662. synapse/storage/databases/main/account_data.py +1073 -0
  663. synapse/storage/databases/main/appservice.py +473 -0
  664. synapse/storage/databases/main/cache.py +912 -0
  665. synapse/storage/databases/main/censor_events.py +226 -0
  666. synapse/storage/databases/main/client_ips.py +816 -0
  667. synapse/storage/databases/main/delayed_events.py +577 -0
  668. synapse/storage/databases/main/deviceinbox.py +1272 -0
  669. synapse/storage/databases/main/devices.py +2579 -0
  670. synapse/storage/databases/main/directory.py +212 -0
  671. synapse/storage/databases/main/e2e_room_keys.py +689 -0
  672. synapse/storage/databases/main/end_to_end_keys.py +1894 -0
  673. synapse/storage/databases/main/event_federation.py +2511 -0
  674. synapse/storage/databases/main/event_push_actions.py +1936 -0
  675. synapse/storage/databases/main/events.py +3765 -0
  676. synapse/storage/databases/main/events_bg_updates.py +2910 -0
  677. synapse/storage/databases/main/events_forward_extremities.py +126 -0
  678. synapse/storage/databases/main/events_worker.py +2787 -0
  679. synapse/storage/databases/main/experimental_features.py +130 -0
  680. synapse/storage/databases/main/filtering.py +231 -0
  681. synapse/storage/databases/main/keys.py +291 -0
  682. synapse/storage/databases/main/lock.py +554 -0
  683. synapse/storage/databases/main/media_repository.py +1068 -0
  684. synapse/storage/databases/main/metrics.py +461 -0
  685. synapse/storage/databases/main/monthly_active_users.py +443 -0
  686. synapse/storage/databases/main/openid.py +60 -0
  687. synapse/storage/databases/main/presence.py +509 -0
  688. synapse/storage/databases/main/profile.py +539 -0
  689. synapse/storage/databases/main/purge_events.py +521 -0
  690. synapse/storage/databases/main/push_rule.py +970 -0
  691. synapse/storage/databases/main/pusher.py +793 -0
  692. synapse/storage/databases/main/receipts.py +1341 -0
  693. synapse/storage/databases/main/registration.py +3071 -0
  694. synapse/storage/databases/main/rejections.py +37 -0
  695. synapse/storage/databases/main/relations.py +1116 -0
  696. synapse/storage/databases/main/room.py +2779 -0
  697. synapse/storage/databases/main/roommember.py +2132 -0
  698. synapse/storage/databases/main/search.py +939 -0
  699. synapse/storage/databases/main/session.py +152 -0
  700. synapse/storage/databases/main/signatures.py +94 -0
  701. synapse/storage/databases/main/sliding_sync.py +842 -0
  702. synapse/storage/databases/main/state.py +1002 -0
  703. synapse/storage/databases/main/state_deltas.py +360 -0
  704. synapse/storage/databases/main/stats.py +789 -0
  705. synapse/storage/databases/main/stream.py +2589 -0
  706. synapse/storage/databases/main/tags.py +360 -0
  707. synapse/storage/databases/main/task_scheduler.py +225 -0
  708. synapse/storage/databases/main/thread_subscriptions.py +589 -0
  709. synapse/storage/databases/main/transactions.py +676 -0
  710. synapse/storage/databases/main/ui_auth.py +420 -0
  711. synapse/storage/databases/main/user_directory.py +1330 -0
  712. synapse/storage/databases/main/user_erasure_store.py +117 -0
  713. synapse/storage/databases/state/__init__.py +22 -0
  714. synapse/storage/databases/state/bg_updates.py +497 -0
  715. synapse/storage/databases/state/deletion.py +557 -0
  716. synapse/storage/databases/state/store.py +948 -0
  717. synapse/storage/engines/__init__.py +70 -0
  718. synapse/storage/engines/_base.py +154 -0
  719. synapse/storage/engines/postgres.py +261 -0
  720. synapse/storage/engines/sqlite.py +199 -0
  721. synapse/storage/invite_rule.py +152 -0
  722. synapse/storage/keys.py +40 -0
  723. synapse/storage/prepare_database.py +730 -0
  724. synapse/storage/push_rule.py +28 -0
  725. synapse/storage/roommember.py +88 -0
  726. synapse/storage/schema/README.md +4 -0
  727. synapse/storage/schema/__init__.py +186 -0
  728. synapse/storage/schema/common/delta/25/00background_updates.sql +40 -0
  729. synapse/storage/schema/common/delta/35/00background_updates_add_col.sql +36 -0
  730. synapse/storage/schema/common/delta/58/00background_update_ordering.sql +38 -0
  731. synapse/storage/schema/common/full_schemas/72/full.sql.postgres +8 -0
  732. synapse/storage/schema/common/full_schemas/72/full.sql.sqlite +6 -0
  733. synapse/storage/schema/common/schema_version.sql +60 -0
  734. synapse/storage/schema/main/delta/12/v12.sql +82 -0
  735. synapse/storage/schema/main/delta/13/v13.sql +38 -0
  736. synapse/storage/schema/main/delta/14/v14.sql +42 -0
  737. synapse/storage/schema/main/delta/15/appservice_txns.sql +50 -0
  738. synapse/storage/schema/main/delta/15/presence_indices.sql +2 -0
  739. synapse/storage/schema/main/delta/15/v15.sql +24 -0
  740. synapse/storage/schema/main/delta/16/events_order_index.sql +4 -0
  741. synapse/storage/schema/main/delta/16/remote_media_cache_index.sql +2 -0
  742. synapse/storage/schema/main/delta/16/remove_duplicates.sql +9 -0
  743. synapse/storage/schema/main/delta/16/room_alias_index.sql +3 -0
  744. synapse/storage/schema/main/delta/16/unique_constraints.sql +72 -0
  745. synapse/storage/schema/main/delta/16/users.sql +56 -0
  746. synapse/storage/schema/main/delta/17/drop_indexes.sql +37 -0
  747. synapse/storage/schema/main/delta/17/server_keys.sql +43 -0
  748. synapse/storage/schema/main/delta/17/user_threepids.sql +9 -0
  749. synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql +51 -0
  750. synapse/storage/schema/main/delta/19/event_index.sql +38 -0
  751. synapse/storage/schema/main/delta/20/dummy.sql +1 -0
  752. synapse/storage/schema/main/delta/20/pushers.py +93 -0
  753. synapse/storage/schema/main/delta/21/end_to_end_keys.sql +53 -0
  754. synapse/storage/schema/main/delta/21/receipts.sql +57 -0
  755. synapse/storage/schema/main/delta/22/receipts_index.sql +41 -0
  756. synapse/storage/schema/main/delta/22/user_threepids_unique.sql +19 -0
  757. synapse/storage/schema/main/delta/24/stats_reporting.sql +37 -0
  758. synapse/storage/schema/main/delta/25/fts.py +81 -0
  759. synapse/storage/schema/main/delta/25/guest_access.sql +44 -0
  760. synapse/storage/schema/main/delta/25/history_visibility.sql +44 -0
  761. synapse/storage/schema/main/delta/25/tags.sql +57 -0
  762. synapse/storage/schema/main/delta/26/account_data.sql +36 -0
  763. synapse/storage/schema/main/delta/27/account_data.sql +55 -0
  764. synapse/storage/schema/main/delta/27/forgotten_memberships.sql +45 -0
  765. synapse/storage/schema/main/delta/27/ts.py +61 -0
  766. synapse/storage/schema/main/delta/28/event_push_actions.sql +46 -0
  767. synapse/storage/schema/main/delta/28/events_room_stream.sql +39 -0
  768. synapse/storage/schema/main/delta/28/public_roms_index.sql +39 -0
  769. synapse/storage/schema/main/delta/28/receipts_user_id_index.sql +41 -0
  770. synapse/storage/schema/main/delta/28/upgrade_times.sql +40 -0
  771. synapse/storage/schema/main/delta/28/users_is_guest.sql +41 -0
  772. synapse/storage/schema/main/delta/29/push_actions.sql +54 -0
  773. synapse/storage/schema/main/delta/30/alias_creator.sql +35 -0
  774. synapse/storage/schema/main/delta/30/as_users.py +82 -0
  775. synapse/storage/schema/main/delta/30/deleted_pushers.sql +44 -0
  776. synapse/storage/schema/main/delta/30/presence_stream.sql +49 -0
  777. synapse/storage/schema/main/delta/30/public_rooms.sql +42 -0
  778. synapse/storage/schema/main/delta/30/push_rule_stream.sql +57 -0
  779. synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql +43 -0
  780. synapse/storage/schema/main/delta/31/invites.sql +61 -0
  781. synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql +46 -0
  782. synapse/storage/schema/main/delta/31/pushers_0.py +92 -0
  783. synapse/storage/schema/main/delta/31/pushers_index.sql +41 -0
  784. synapse/storage/schema/main/delta/31/search_update.py +65 -0
  785. synapse/storage/schema/main/delta/32/events.sql +35 -0
  786. synapse/storage/schema/main/delta/32/openid.sql +9 -0
  787. synapse/storage/schema/main/delta/32/pusher_throttle.sql +42 -0
  788. synapse/storage/schema/main/delta/32/remove_indices.sql +52 -0
  789. synapse/storage/schema/main/delta/32/reports.sql +44 -0
  790. synapse/storage/schema/main/delta/33/access_tokens_device_index.sql +36 -0
  791. synapse/storage/schema/main/delta/33/devices.sql +40 -0
  792. synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql +38 -0
  793. synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql +39 -0
  794. synapse/storage/schema/main/delta/33/event_fields.py +61 -0
  795. synapse/storage/schema/main/delta/33/remote_media_ts.py +43 -0
  796. synapse/storage/schema/main/delta/33/user_ips_index.sql +36 -0
  797. synapse/storage/schema/main/delta/34/appservice_stream.sql +42 -0
  798. synapse/storage/schema/main/delta/34/cache_stream.py +50 -0
  799. synapse/storage/schema/main/delta/34/device_inbox.sql +43 -0
  800. synapse/storage/schema/main/delta/34/push_display_name_rename.sql +39 -0
  801. synapse/storage/schema/main/delta/34/received_txn_purge.py +36 -0
  802. synapse/storage/schema/main/delta/35/contains_url.sql +36 -0
  803. synapse/storage/schema/main/delta/35/device_outbox.sql +58 -0
  804. synapse/storage/schema/main/delta/35/device_stream_id.sql +40 -0
  805. synapse/storage/schema/main/delta/35/event_push_actions_index.sql +36 -0
  806. synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql +52 -0
  807. synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql +56 -0
  808. synapse/storage/schema/main/delta/36/readd_public_rooms.sql +45 -0
  809. synapse/storage/schema/main/delta/37/remove_auth_idx.py +89 -0
  810. synapse/storage/schema/main/delta/37/user_threepids.sql +71 -0
  811. synapse/storage/schema/main/delta/38/postgres_fts_gist.sql +38 -0
  812. synapse/storage/schema/main/delta/39/appservice_room_list.sql +48 -0
  813. synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql +35 -0
  814. synapse/storage/schema/main/delta/39/event_push_index.sql +36 -0
  815. synapse/storage/schema/main/delta/39/federation_out_position.sql +41 -0
  816. synapse/storage/schema/main/delta/39/membership_profile.sql +39 -0
  817. synapse/storage/schema/main/delta/40/current_state_idx.sql +36 -0
  818. synapse/storage/schema/main/delta/40/device_inbox.sql +40 -0
  819. synapse/storage/schema/main/delta/40/device_list_streams.sql +79 -0
  820. synapse/storage/schema/main/delta/40/event_push_summary.sql +57 -0
  821. synapse/storage/schema/main/delta/40/pushers.sql +58 -0
  822. synapse/storage/schema/main/delta/41/device_list_stream_idx.sql +36 -0
  823. synapse/storage/schema/main/delta/41/device_outbound_index.sql +35 -0
  824. synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql +36 -0
  825. synapse/storage/schema/main/delta/41/ratelimit.sql +41 -0
  826. synapse/storage/schema/main/delta/42/current_state_delta.sql +48 -0
  827. synapse/storage/schema/main/delta/42/device_list_last_id.sql +52 -0
  828. synapse/storage/schema/main/delta/42/event_auth_state_only.sql +36 -0
  829. synapse/storage/schema/main/delta/42/user_dir.py +88 -0
  830. synapse/storage/schema/main/delta/43/blocked_rooms.sql +40 -0
  831. synapse/storage/schema/main/delta/43/quarantine_media.sql +36 -0
  832. synapse/storage/schema/main/delta/43/url_cache.sql +35 -0
  833. synapse/storage/schema/main/delta/43/user_share.sql +52 -0
  834. synapse/storage/schema/main/delta/44/expire_url_cache.sql +60 -0
  835. synapse/storage/schema/main/delta/45/group_server.sql +186 -0
  836. synapse/storage/schema/main/delta/45/profile_cache.sql +47 -0
  837. synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql +36 -0
  838. synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql +54 -0
  839. synapse/storage/schema/main/delta/46/group_server.sql +51 -0
  840. synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql +43 -0
  841. synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql +54 -0
  842. synapse/storage/schema/main/delta/46/user_dir_typos.sql +43 -0
  843. synapse/storage/schema/main/delta/47/last_access_media.sql +35 -0
  844. synapse/storage/schema/main/delta/47/postgres_fts_gin.sql +36 -0
  845. synapse/storage/schema/main/delta/47/push_actions_staging.sql +47 -0
  846. synapse/storage/schema/main/delta/48/add_user_consent.sql +37 -0
  847. synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql +36 -0
  848. synapse/storage/schema/main/delta/48/deactivated_users.sql +44 -0
  849. synapse/storage/schema/main/delta/48/group_unique_indexes.py +67 -0
  850. synapse/storage/schema/main/delta/48/groups_joinable.sql +41 -0
  851. synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql +39 -0
  852. synapse/storage/schema/main/delta/49/add_user_daily_visits.sql +40 -0
  853. synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql +36 -0
  854. synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql +38 -0
  855. synapse/storage/schema/main/delta/50/erasure_store.sql +40 -0
  856. synapse/storage/schema/main/delta/50/make_event_content_nullable.py +102 -0
  857. synapse/storage/schema/main/delta/51/e2e_room_keys.sql +58 -0
  858. synapse/storage/schema/main/delta/51/monthly_active_users.sql +46 -0
  859. synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql +38 -0
  860. synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql +55 -0
  861. synapse/storage/schema/main/delta/52/e2e_room_keys.sql +72 -0
  862. synapse/storage/schema/main/delta/53/add_user_type_to_users.sql +38 -0
  863. synapse/storage/schema/main/delta/53/drop_sent_transactions.sql +35 -0
  864. synapse/storage/schema/main/delta/53/event_format_version.sql +35 -0
  865. synapse/storage/schema/main/delta/53/user_dir_populate.sql +49 -0
  866. synapse/storage/schema/main/delta/53/user_ips_index.sql +49 -0
  867. synapse/storage/schema/main/delta/53/user_share.sql +63 -0
  868. synapse/storage/schema/main/delta/53/user_threepid_id.sql +48 -0
  869. synapse/storage/schema/main/delta/53/users_in_public_rooms.sql +47 -0
  870. synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql +49 -0
  871. synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql +42 -0
  872. synapse/storage/schema/main/delta/54/delete_forward_extremities.sql +42 -0
  873. synapse/storage/schema/main/delta/54/drop_legacy_tables.sql +49 -0
  874. synapse/storage/schema/main/delta/54/drop_presence_list.sql +35 -0
  875. synapse/storage/schema/main/delta/54/relations.sql +46 -0
  876. synapse/storage/schema/main/delta/54/stats.sql +99 -0
  877. synapse/storage/schema/main/delta/54/stats2.sql +47 -0
  878. synapse/storage/schema/main/delta/55/access_token_expiry.sql +37 -0
  879. synapse/storage/schema/main/delta/55/track_threepid_validations.sql +50 -0
  880. synapse/storage/schema/main/delta/55/users_alter_deactivated.sql +38 -0
  881. synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql +39 -0
  882. synapse/storage/schema/main/delta/56/current_state_events_membership.sql +41 -0
  883. synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql +43 -0
  884. synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql +44 -0
  885. synapse/storage/schema/main/delta/56/destinations_failure_ts.sql +44 -0
  886. synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres +18 -0
  887. synapse/storage/schema/main/delta/56/device_stream_id_insert.sql +39 -0
  888. synapse/storage/schema/main/delta/56/devices_last_seen.sql +43 -0
  889. synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql +39 -0
  890. synapse/storage/schema/main/delta/56/event_expiry.sql +40 -0
  891. synapse/storage/schema/main/delta/56/event_labels.sql +49 -0
  892. synapse/storage/schema/main/delta/56/event_labels_background_update.sql +36 -0
  893. synapse/storage/schema/main/delta/56/fix_room_keys_index.sql +37 -0
  894. synapse/storage/schema/main/delta/56/hidden_devices.sql +37 -0
  895. synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite +42 -0
  896. synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql +48 -0
  897. synapse/storage/schema/main/delta/56/public_room_list_idx.sql +35 -0
  898. synapse/storage/schema/main/delta/56/redaction_censor.sql +35 -0
  899. synapse/storage/schema/main/delta/56/redaction_censor2.sql +41 -0
  900. synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres +25 -0
  901. synapse/storage/schema/main/delta/56/redaction_censor4.sql +35 -0
  902. synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql +38 -0
  903. synapse/storage/schema/main/delta/56/room_key_etag.sql +36 -0
  904. synapse/storage/schema/main/delta/56/room_membership_idx.sql +37 -0
  905. synapse/storage/schema/main/delta/56/room_retention.sql +52 -0
  906. synapse/storage/schema/main/delta/56/signing_keys.sql +75 -0
  907. synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql +41 -0
  908. synapse/storage/schema/main/delta/56/stats_separated.sql +175 -0
  909. synapse/storage/schema/main/delta/56/unique_user_filter_index.py +46 -0
  910. synapse/storage/schema/main/delta/56/user_external_ids.sql +43 -0
  911. synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql +36 -0
  912. synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql +41 -0
  913. synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql +44 -0
  914. synapse/storage/schema/main/delta/57/local_current_membership.py +111 -0
  915. synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql +40 -0
  916. synapse/storage/schema/main/delta/57/rooms_version_column.sql +43 -0
  917. synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres +35 -0
  918. synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite +22 -0
  919. synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres +39 -0
  920. synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite +23 -0
  921. synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql +41 -0
  922. synapse/storage/schema/main/delta/58/03persist_ui_auth.sql +55 -0
  923. synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres +30 -0
  924. synapse/storage/schema/main/delta/58/06dlols_unique_idx.py +83 -0
  925. synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres +33 -0
  926. synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite +44 -0
  927. synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql +44 -0
  928. synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres +18 -0
  929. synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite +18 -0
  930. synapse/storage/schema/main/delta/58/09shadow_ban.sql +37 -0
  931. synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql +47 -0
  932. synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql +41 -0
  933. synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql +41 -0
  934. synapse/storage/schema/main/delta/58/11dehydration.sql +39 -0
  935. synapse/storage/schema/main/delta/58/11fallback.sql +43 -0
  936. synapse/storage/schema/main/delta/58/11user_id_seq.py +38 -0
  937. synapse/storage/schema/main/delta/58/12room_stats.sql +51 -0
  938. synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql +36 -0
  939. synapse/storage/schema/main/delta/58/14events_instance_name.sql +35 -0
  940. synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres +28 -0
  941. synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql +61 -0
  942. synapse/storage/schema/main/delta/58/15unread_count.sql +45 -0
  943. synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql +41 -0
  944. synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql +40 -0
  945. synapse/storage/schema/main/delta/58/18stream_positions.sql +41 -0
  946. synapse/storage/schema/main/delta/58/19instance_map.sql.postgres +25 -0
  947. synapse/storage/schema/main/delta/58/19txn_id.sql +59 -0
  948. synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql +36 -0
  949. synapse/storage/schema/main/delta/58/20user_daily_visits.sql +37 -0
  950. synapse/storage/schema/main/delta/58/21as_device_stream.sql +36 -0
  951. synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql +1 -0
  952. synapse/storage/schema/main/delta/58/22puppet_token.sql +36 -0
  953. synapse/storage/schema/main/delta/58/22users_have_local_media.sql +2 -0
  954. synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql +36 -0
  955. synapse/storage/schema/main/delta/58/24drop_event_json_index.sql +38 -0
  956. synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql +36 -0
  957. synapse/storage/schema/main/delta/58/26access_token_last_validated.sql +37 -0
  958. synapse/storage/schema/main/delta/58/27local_invites.sql +37 -0
  959. synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres +16 -0
  960. synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite +62 -0
  961. synapse/storage/schema/main/delta/59/01ignored_user.py +85 -0
  962. synapse/storage/schema/main/delta/59/02shard_send_to_device.sql +37 -0
  963. synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres +25 -0
  964. synapse/storage/schema/main/delta/59/04_event_auth_chains.sql +71 -0
  965. synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres +16 -0
  966. synapse/storage/schema/main/delta/59/04drop_account_data.sql +36 -0
  967. synapse/storage/schema/main/delta/59/05cache_invalidation.sql +36 -0
  968. synapse/storage/schema/main/delta/59/06chain_cover_index.sql +36 -0
  969. synapse/storage/schema/main/delta/59/06shard_account_data.sql +39 -0
  970. synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres +32 -0
  971. synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql +37 -0
  972. synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql +39 -0
  973. synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql +39 -0
  974. synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql +45 -0
  975. synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql +36 -0
  976. synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql +39 -0
  977. synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres +22 -0
  978. synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql +37 -0
  979. synapse/storage/schema/main/delta/59/12presence_stream_instance.sql +37 -0
  980. synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres +20 -0
  981. synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql +53 -0
  982. synapse/storage/schema/main/delta/59/14refresh_tokens.sql +53 -0
  983. synapse/storage/schema/main/delta/59/15locks.sql +56 -0
  984. synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql +51 -0
  985. synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres +45 -0
  986. synapse/storage/schema/main/delta/60/02change_stream_ordering_columns.sql.postgres +30 -0
  987. synapse/storage/schema/main/delta/61/01change_appservices_txns.sql.postgres +23 -0
  988. synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql +68 -0
  989. synapse/storage/schema/main/delta/61/02drop_redundant_room_depth_index.sql +37 -0
  990. synapse/storage/schema/main/delta/61/03recreate_min_depth.py +74 -0
  991. synapse/storage/schema/main/delta/62/01insertion_event_extremities.sql +43 -0
  992. synapse/storage/schema/main/delta/63/01create_registration_tokens.sql +42 -0
  993. synapse/storage/schema/main/delta/63/02delete_unlinked_email_pushers.sql +39 -0
  994. synapse/storage/schema/main/delta/63/02populate-rooms-creator.sql +36 -0
  995. synapse/storage/schema/main/delta/63/03session_store.sql +42 -0
  996. synapse/storage/schema/main/delta/63/04add_presence_stream_not_offline_index.sql +37 -0
  997. synapse/storage/schema/main/delta/64/01msc2716_chunk_to_batch_rename.sql.postgres +23 -0
  998. synapse/storage/schema/main/delta/64/01msc2716_chunk_to_batch_rename.sql.sqlite +37 -0
  999. synapse/storage/schema/main/delta/65/01msc2716_insertion_event_edges.sql +38 -0
  1000. synapse/storage/schema/main/delta/65/03remove_hidden_devices_from_device_inbox.sql +41 -0
  1001. synapse/storage/schema/main/delta/65/04_local_group_updates.sql +37 -0
  1002. synapse/storage/schema/main/delta/65/05_remove_room_stats_historical_and_user_stats_historical.sql +38 -0
  1003. synapse/storage/schema/main/delta/65/06remove_deleted_devices_from_device_inbox.sql +53 -0
  1004. synapse/storage/schema/main/delta/65/07_arbitrary_relations.sql +37 -0
  1005. synapse/storage/schema/main/delta/65/08_device_inbox_background_updates.sql +37 -0
  1006. synapse/storage/schema/main/delta/65/10_expirable_refresh_tokens.sql +47 -0
  1007. synapse/storage/schema/main/delta/65/11_devices_auth_provider_session.sql +46 -0
  1008. synapse/storage/schema/main/delta/67/01drop_public_room_list_stream.sql +37 -0
  1009. synapse/storage/schema/main/delta/68/01event_columns.sql +45 -0
  1010. synapse/storage/schema/main/delta/68/02_msc2409_add_device_id_appservice_stream_type.sql +40 -0
  1011. synapse/storage/schema/main/delta/68/03_delete_account_data_for_deactivated_accounts.sql +39 -0
  1012. synapse/storage/schema/main/delta/68/04_refresh_tokens_index_next_token_id.sql +47 -0
  1013. synapse/storage/schema/main/delta/68/04partial_state_rooms.sql +60 -0
  1014. synapse/storage/schema/main/delta/68/05_delete_non_strings_from_event_search.sql.sqlite +22 -0
  1015. synapse/storage/schema/main/delta/68/05partial_state_rooms_triggers.py +80 -0
  1016. synapse/storage/schema/main/delta/68/06_msc3202_add_device_list_appservice_stream_type.sql +42 -0
  1017. synapse/storage/schema/main/delta/69/01as_txn_seq.py +54 -0
  1018. synapse/storage/schema/main/delta/69/01device_list_oubound_by_room.sql +57 -0
  1019. synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql +37 -0
  1020. synapse/storage/schema/main/delta/70/01clean_table_purged_rooms.sql +39 -0
  1021. synapse/storage/schema/main/delta/71/01rebuild_event_edges.sql.postgres +43 -0
  1022. synapse/storage/schema/main/delta/71/01rebuild_event_edges.sql.sqlite +47 -0
  1023. synapse/storage/schema/main/delta/71/01remove_noop_background_updates.sql +80 -0
  1024. synapse/storage/schema/main/delta/71/02event_push_summary_unique.sql +37 -0
  1025. synapse/storage/schema/main/delta/72/01add_room_type_to_state_stats.sql +38 -0
  1026. synapse/storage/schema/main/delta/72/01event_push_summary_receipt.sql +54 -0
  1027. synapse/storage/schema/main/delta/72/02event_push_actions_index.sql +38 -0
  1028. synapse/storage/schema/main/delta/72/03bg_populate_events_columns.py +57 -0
  1029. synapse/storage/schema/main/delta/72/03drop_event_reference_hashes.sql +36 -0
  1030. synapse/storage/schema/main/delta/72/03remove_groups.sql +50 -0
  1031. synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.postgres +17 -0
  1032. synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.sqlite +40 -0
  1033. synapse/storage/schema/main/delta/72/05receipts_event_stream_ordering.sql +38 -0
  1034. synapse/storage/schema/main/delta/72/05remove_unstable_private_read_receipts.sql +38 -0
  1035. synapse/storage/schema/main/delta/72/06add_consent_ts_to_users.sql +35 -0
  1036. synapse/storage/schema/main/delta/72/06thread_notifications.sql +49 -0
  1037. synapse/storage/schema/main/delta/72/07force_update_current_state_events_membership.py +67 -0
  1038. synapse/storage/schema/main/delta/72/07thread_receipts.sql.postgres +30 -0
  1039. synapse/storage/schema/main/delta/72/07thread_receipts.sql.sqlite +70 -0
  1040. synapse/storage/schema/main/delta/72/08begin_cache_invalidation_seq_at_2.sql.postgres +23 -0
  1041. synapse/storage/schema/main/delta/72/08thread_receipts.sql +39 -0
  1042. synapse/storage/schema/main/delta/72/09partial_indices.sql.sqlite +56 -0
  1043. synapse/storage/schema/main/delta/73/01event_failed_pull_attempts.sql +48 -0
  1044. synapse/storage/schema/main/delta/73/02add_pusher_enabled.sql +35 -0
  1045. synapse/storage/schema/main/delta/73/02room_id_indexes_for_purging.sql +41 -0
  1046. synapse/storage/schema/main/delta/73/03pusher_device_id.sql +39 -0
  1047. synapse/storage/schema/main/delta/73/03users_approved_column.sql +39 -0
  1048. synapse/storage/schema/main/delta/73/04partial_join_details.sql +42 -0
  1049. synapse/storage/schema/main/delta/73/04pending_device_list_updates.sql +47 -0
  1050. synapse/storage/schema/main/delta/73/05old_push_actions.sql.postgres +22 -0
  1051. synapse/storage/schema/main/delta/73/05old_push_actions.sql.sqlite +24 -0
  1052. synapse/storage/schema/main/delta/73/06thread_notifications_thread_id_idx.sql +42 -0
  1053. synapse/storage/schema/main/delta/73/08thread_receipts_non_null.sql.postgres +23 -0
  1054. synapse/storage/schema/main/delta/73/08thread_receipts_non_null.sql.sqlite +76 -0
  1055. synapse/storage/schema/main/delta/73/09partial_joined_via_destination.sql +37 -0
  1056. synapse/storage/schema/main/delta/73/09threads_table.sql +49 -0
  1057. synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py +71 -0
  1058. synapse/storage/schema/main/delta/73/10login_tokens.sql +54 -0
  1059. synapse/storage/schema/main/delta/73/11event_search_room_id_n_distinct.sql.postgres +33 -0
  1060. synapse/storage/schema/main/delta/73/12refactor_device_list_outbound_pokes.sql +72 -0
  1061. synapse/storage/schema/main/delta/73/13add_device_lists_index.sql +39 -0
  1062. synapse/storage/schema/main/delta/73/20_un_partial_stated_room_stream.sql +51 -0
  1063. synapse/storage/schema/main/delta/73/21_un_partial_stated_room_stream_seq.sql.postgres +20 -0
  1064. synapse/storage/schema/main/delta/73/22_rebuild_user_dir_stats.sql +48 -0
  1065. synapse/storage/schema/main/delta/73/22_un_partial_stated_event_stream.sql +53 -0
  1066. synapse/storage/schema/main/delta/73/23_fix_thread_index.sql +52 -0
  1067. synapse/storage/schema/main/delta/73/23_un_partial_stated_room_stream_seq.sql.postgres +20 -0
  1068. synapse/storage/schema/main/delta/73/24_events_jump_to_date_index.sql +36 -0
  1069. synapse/storage/schema/main/delta/73/25drop_presence.sql +36 -0
  1070. synapse/storage/schema/main/delta/74/01_user_directory_stale_remote_users.sql +58 -0
  1071. synapse/storage/schema/main/delta/74/02_set_device_id_for_pushers_bg_update.sql +38 -0
  1072. synapse/storage/schema/main/delta/74/03_membership_tables_event_stream_ordering.sql.postgres +29 -0
  1073. synapse/storage/schema/main/delta/74/03_membership_tables_event_stream_ordering.sql.sqlite +23 -0
  1074. synapse/storage/schema/main/delta/74/03_room_membership_index.sql +38 -0
  1075. synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql +36 -0
  1076. synapse/storage/schema/main/delta/74/04_membership_tables_event_stream_ordering_triggers.py +87 -0
  1077. synapse/storage/schema/main/delta/74/05_events_txn_id_device_id.sql +72 -0
  1078. synapse/storage/schema/main/delta/74/90COMMENTS_destinations.sql.postgres +52 -0
  1079. synapse/storage/schema/main/delta/76/01_add_profiles_full_user_id_column.sql +39 -0
  1080. synapse/storage/schema/main/delta/76/02_add_user_filters_full_user_id_column.sql +39 -0
  1081. synapse/storage/schema/main/delta/76/03_per_user_experimental_features.sql +46 -0
  1082. synapse/storage/schema/main/delta/76/04_add_room_forgetter.sql +43 -0
  1083. synapse/storage/schema/main/delta/77/01_add_profiles_not_valid_check.sql.postgres +16 -0
  1084. synapse/storage/schema/main/delta/77/02_add_user_filters_not_valid_check.sql.postgres +16 -0
  1085. synapse/storage/schema/main/delta/77/03bg_populate_full_user_id_profiles.sql +35 -0
  1086. synapse/storage/schema/main/delta/77/04bg_populate_full_user_id_user_filters.sql +35 -0
  1087. synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql +67 -0
  1088. synapse/storage/schema/main/delta/77/06thread_notifications_not_null.sql.sqlite +102 -0
  1089. synapse/storage/schema/main/delta/77/06thread_notifications_not_null_event_push_actions.sql.postgres +27 -0
  1090. synapse/storage/schema/main/delta/77/06thread_notifications_not_null_event_push_actions_staging.sql.postgres +27 -0
  1091. synapse/storage/schema/main/delta/77/06thread_notifications_not_null_event_push_summary.sql.postgres +29 -0
  1092. synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql +39 -0
  1093. synapse/storage/schema/main/delta/78/01_validate_and_update_profiles.py +99 -0
  1094. synapse/storage/schema/main/delta/78/02_validate_and_update_user_filters.py +100 -0
  1095. synapse/storage/schema/main/delta/78/03_remove_unused_indexes_user_filters.py +72 -0
  1096. synapse/storage/schema/main/delta/78/03event_extremities_constraints.py +65 -0
  1097. synapse/storage/schema/main/delta/78/04_add_full_user_id_index_user_filters.py +32 -0
  1098. synapse/storage/schema/main/delta/79/03_read_write_locks_triggers.sql.postgres +102 -0
  1099. synapse/storage/schema/main/delta/79/03_read_write_locks_triggers.sql.sqlite +72 -0
  1100. synapse/storage/schema/main/delta/79/04_mitigate_stream_ordering_update_race.py +70 -0
  1101. synapse/storage/schema/main/delta/79/05_read_write_locks_triggers.sql.postgres +69 -0
  1102. synapse/storage/schema/main/delta/79/05_read_write_locks_triggers.sql.sqlite +65 -0
  1103. synapse/storage/schema/main/delta/80/01_users_alter_locked.sql +35 -0
  1104. synapse/storage/schema/main/delta/80/02_read_write_locks_unlogged.sql.postgres +30 -0
  1105. synapse/storage/schema/main/delta/80/02_scheduled_tasks.sql +47 -0
  1106. synapse/storage/schema/main/delta/80/03_read_write_locks_triggers.sql.postgres +37 -0
  1107. synapse/storage/schema/main/delta/80/04_read_write_locks_deadlock.sql.postgres +71 -0
  1108. synapse/storage/schema/main/delta/82/02_scheduled_tasks_index.sql +35 -0
  1109. synapse/storage/schema/main/delta/82/04_add_indices_for_purging_rooms.sql +39 -0
  1110. synapse/storage/schema/main/delta/82/05gaps.sql +44 -0
  1111. synapse/storage/schema/main/delta/83/01_drop_old_tables.sql +43 -0
  1112. synapse/storage/schema/main/delta/83/03_instance_name_receipts.sql.sqlite +17 -0
  1113. synapse/storage/schema/main/delta/83/05_cross_signing_key_update_grant.sql +34 -0
  1114. synapse/storage/schema/main/delta/83/06_event_push_summary_room.sql +36 -0
  1115. synapse/storage/schema/main/delta/84/01_auth_links_stats.sql.postgres +20 -0
  1116. synapse/storage/schema/main/delta/84/02_auth_links_index.sql +16 -0
  1117. synapse/storage/schema/main/delta/84/03_auth_links_analyze.sql.postgres +16 -0
  1118. synapse/storage/schema/main/delta/84/04_access_token_index.sql +15 -0
  1119. synapse/storage/schema/main/delta/85/01_add_suspended.sql +14 -0
  1120. synapse/storage/schema/main/delta/85/02_add_instance_names.sql +27 -0
  1121. synapse/storage/schema/main/delta/85/03_new_sequences.sql.postgres +54 -0
  1122. synapse/storage/schema/main/delta/85/04_cleanup_device_federation_outbox.sql +15 -0
  1123. synapse/storage/schema/main/delta/85/05_add_instance_names_converted_pos.sql +16 -0
  1124. synapse/storage/schema/main/delta/85/06_add_room_reports.sql +20 -0
  1125. synapse/storage/schema/main/delta/86/01_authenticate_media.sql +15 -0
  1126. synapse/storage/schema/main/delta/86/02_receipts_event_id_index.sql +15 -0
  1127. synapse/storage/schema/main/delta/87/01_sliding_sync_memberships.sql +169 -0
  1128. synapse/storage/schema/main/delta/87/02_per_connection_state.sql +81 -0
  1129. synapse/storage/schema/main/delta/87/03_current_state_index.sql +19 -0
  1130. synapse/storage/schema/main/delta/88/01_add_delayed_events.sql +43 -0
  1131. synapse/storage/schema/main/delta/88/01_custom_profile_fields.sql +15 -0
  1132. synapse/storage/schema/main/delta/88/02_fix_sliding_sync_membership_snapshots_forgotten_column.sql +21 -0
  1133. synapse/storage/schema/main/delta/88/03_add_otk_ts_added_index.sql +18 -0
  1134. synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql +18 -0
  1135. synapse/storage/schema/main/delta/88/05_drop_old_otks.sql.postgres +19 -0
  1136. synapse/storage/schema/main/delta/88/05_drop_old_otks.sql.sqlite +19 -0
  1137. synapse/storage/schema/main/delta/88/05_sliding_sync_room_config_index.sql +20 -0
  1138. synapse/storage/schema/main/delta/88/06_events_received_ts_index.sql +17 -0
  1139. synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql +15 -0
  1140. synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql +16 -0
  1141. synapse/storage/schema/main/delta/91/01_media_hash.sql +28 -0
  1142. synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres +16 -0
  1143. synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite +16 -0
  1144. synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql +17 -0
  1145. synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql +16 -0
  1146. synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql +59 -0
  1147. synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres +19 -0
  1148. synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql +17 -0
  1149. synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres +18 -0
  1150. synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql +16 -0
  1151. synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres +24 -0
  1152. synapse/storage/schema/main/delta/92/07_add_user_reports.sql +22 -0
  1153. synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql +15 -0
  1154. synapse/storage/schema/main/delta/92/08_room_ban_redactions.sql +21 -0
  1155. synapse/storage/schema/main/delta/92/08_thread_subscriptions_seq_fixup.sql.postgres +19 -0
  1156. synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql +20 -0
  1157. synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql.postgres +18 -0
  1158. synapse/storage/schema/main/delta/93/01_add_delayed_events.sql +15 -0
  1159. synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql +60 -0
  1160. synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql +27 -0
  1161. synapse/storage/schema/main/full_schemas/72/full.sql.postgres +1344 -0
  1162. synapse/storage/schema/main/full_schemas/72/full.sql.sqlite +646 -0
  1163. synapse/storage/schema/state/delta/23/drop_state_index.sql +35 -0
  1164. synapse/storage/schema/state/delta/32/remove_state_indices.sql +38 -0
  1165. synapse/storage/schema/state/delta/35/add_state_index.sql +36 -0
  1166. synapse/storage/schema/state/delta/35/state.sql +41 -0
  1167. synapse/storage/schema/state/delta/35/state_dedupe.sql +36 -0
  1168. synapse/storage/schema/state/delta/47/state_group_seq.py +38 -0
  1169. synapse/storage/schema/state/delta/56/state_group_room_idx.sql +36 -0
  1170. synapse/storage/schema/state/delta/61/02state_groups_state_n_distinct.sql.postgres +34 -0
  1171. synapse/storage/schema/state/delta/70/08_state_group_edges_unique.sql +36 -0
  1172. synapse/storage/schema/state/delta/89/01_state_groups_deletion.sql +39 -0
  1173. synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql +16 -0
  1174. synapse/storage/schema/state/delta/90/03_remove_old_deletion_bg_update.sql +15 -0
  1175. synapse/storage/schema/state/full_schemas/72/full.sql.postgres +30 -0
  1176. synapse/storage/schema/state/full_schemas/72/full.sql.sqlite +20 -0
  1177. synapse/storage/types.py +183 -0
  1178. synapse/storage/util/__init__.py +20 -0
  1179. synapse/storage/util/id_generators.py +928 -0
  1180. synapse/storage/util/partial_state_events_tracker.py +194 -0
  1181. synapse/storage/util/sequence.py +315 -0
  1182. synapse/streams/__init__.py +43 -0
  1183. synapse/streams/config.py +91 -0
  1184. synapse/streams/events.py +203 -0
  1185. synapse/synapse_rust/__init__.pyi +3 -0
  1186. synapse/synapse_rust/acl.pyi +20 -0
  1187. synapse/synapse_rust/events.pyi +136 -0
  1188. synapse/synapse_rust/http_client.pyi +32 -0
  1189. synapse/synapse_rust/push.pyi +86 -0
  1190. synapse/synapse_rust/rendezvous.pyi +30 -0
  1191. synapse/synapse_rust/segmenter.pyi +1 -0
  1192. synapse/synapse_rust.abi3.so +0 -0
  1193. synapse/types/__init__.py +1600 -0
  1194. synapse/types/handlers/__init__.py +93 -0
  1195. synapse/types/handlers/policy_server.py +16 -0
  1196. synapse/types/handlers/sliding_sync.py +1004 -0
  1197. synapse/types/rest/__init__.py +25 -0
  1198. synapse/types/rest/client/__init__.py +413 -0
  1199. synapse/types/state.py +634 -0
  1200. synapse/types/storage/__init__.py +66 -0
  1201. synapse/util/__init__.py +160 -0
  1202. synapse/util/async_helpers.py +1048 -0
  1203. synapse/util/background_queue.py +142 -0
  1204. synapse/util/batching_queue.py +203 -0
  1205. synapse/util/caches/__init__.py +300 -0
  1206. synapse/util/caches/cached_call.py +143 -0
  1207. synapse/util/caches/deferred_cache.py +530 -0
  1208. synapse/util/caches/descriptors.py +692 -0
  1209. synapse/util/caches/dictionary_cache.py +346 -0
  1210. synapse/util/caches/expiringcache.py +250 -0
  1211. synapse/util/caches/lrucache.py +976 -0
  1212. synapse/util/caches/response_cache.py +323 -0
  1213. synapse/util/caches/stream_change_cache.py +370 -0
  1214. synapse/util/caches/treecache.py +189 -0
  1215. synapse/util/caches/ttlcache.py +197 -0
  1216. synapse/util/cancellation.py +63 -0
  1217. synapse/util/check_dependencies.py +335 -0
  1218. synapse/util/clock.py +592 -0
  1219. synapse/util/daemonize.py +165 -0
  1220. synapse/util/distributor.py +157 -0
  1221. synapse/util/duration.py +117 -0
  1222. synapse/util/events.py +134 -0
  1223. synapse/util/file_consumer.py +164 -0
  1224. synapse/util/frozenutils.py +57 -0
  1225. synapse/util/gai_resolver.py +178 -0
  1226. synapse/util/hash.py +38 -0
  1227. synapse/util/httpresourcetree.py +108 -0
  1228. synapse/util/iterutils.py +190 -0
  1229. synapse/util/json.py +56 -0
  1230. synapse/util/linked_list.py +156 -0
  1231. synapse/util/logcontext.py +46 -0
  1232. synapse/util/logformatter.py +28 -0
  1233. synapse/util/macaroons.py +325 -0
  1234. synapse/util/manhole.py +191 -0
  1235. synapse/util/metrics.py +339 -0
  1236. synapse/util/module_loader.py +116 -0
  1237. synapse/util/msisdn.py +51 -0
  1238. synapse/util/patch_inline_callbacks.py +250 -0
  1239. synapse/util/pydantic_models.py +63 -0
  1240. synapse/util/ratelimitutils.py +422 -0
  1241. synapse/util/retryutils.py +339 -0
  1242. synapse/util/rlimit.py +42 -0
  1243. synapse/util/rust.py +164 -0
  1244. synapse/util/sentinel.py +21 -0
  1245. synapse/util/stringutils.py +293 -0
  1246. synapse/util/task_scheduler.py +494 -0
  1247. synapse/util/templates.py +126 -0
  1248. synapse/util/threepids.py +123 -0
  1249. synapse/util/wheel_timer.py +112 -0
  1250. synapse/visibility.py +869 -0
  1251. synmark/__init__.py +47 -0
  1252. synmark/__main__.py +128 -0
  1253. synmark/suites/__init__.py +9 -0
  1254. synmark/suites/logging.py +154 -0
  1255. synmark/suites/lrucache.py +48 -0
  1256. synmark/suites/lrucache_evict.py +49 -0
  1257. sytest-blacklist +33 -0
  1258. tests/__init__.py +29 -0
  1259. tests/api/__init__.py +0 -0
  1260. tests/api/test_auth.py +548 -0
  1261. tests/api/test_errors.py +46 -0
  1262. tests/api/test_filtering.py +622 -0
  1263. tests/api/test_ratelimiting.py +505 -0
  1264. tests/api/test_urls.py +81 -0
  1265. tests/app/__init__.py +0 -0
  1266. tests/app/test_homeserver_shutdown.py +271 -0
  1267. tests/app/test_homeserver_start.py +49 -0
  1268. tests/app/test_openid_listener.py +139 -0
  1269. tests/app/test_phone_stats_home.py +248 -0
  1270. tests/appservice/__init__.py +20 -0
  1271. tests/appservice/test_api.py +253 -0
  1272. tests/appservice/test_appservice.py +259 -0
  1273. tests/appservice/test_scheduler.py +477 -0
  1274. tests/config/__init__.py +20 -0
  1275. tests/config/test___main__.py +38 -0
  1276. tests/config/test_api.py +146 -0
  1277. tests/config/test_appservice.py +48 -0
  1278. tests/config/test_background_update.py +65 -0
  1279. tests/config/test_base.py +151 -0
  1280. tests/config/test_cache.py +193 -0
  1281. tests/config/test_database.py +42 -0
  1282. tests/config/test_generate.py +72 -0
  1283. tests/config/test_load.py +322 -0
  1284. tests/config/test_oauth_delegation.py +454 -0
  1285. tests/config/test_ratelimiting.py +77 -0
  1286. tests/config/test_registration_config.py +203 -0
  1287. tests/config/test_room_directory.py +203 -0
  1288. tests/config/test_server.py +248 -0
  1289. tests/config/test_tls.py +209 -0
  1290. tests/config/test_util.py +59 -0
  1291. tests/config/test_workers.py +332 -0
  1292. tests/config/utils.py +66 -0
  1293. tests/crypto/__init__.py +20 -0
  1294. tests/crypto/test_event_signing.py +109 -0
  1295. tests/crypto/test_keyring.py +737 -0
  1296. tests/events/__init__.py +0 -0
  1297. tests/events/test_auto_accept_invites.py +827 -0
  1298. tests/events/test_presence_router.py +540 -0
  1299. tests/events/test_snapshot.py +121 -0
  1300. tests/events/test_utils.py +986 -0
  1301. tests/federation/__init__.py +0 -0
  1302. tests/federation/test_complexity.py +257 -0
  1303. tests/federation/test_federation_catch_up.py +585 -0
  1304. tests/federation/test_federation_client.py +316 -0
  1305. tests/federation/test_federation_devices.py +161 -0
  1306. tests/federation/test_federation_media.py +295 -0
  1307. tests/federation/test_federation_out_of_band_membership.py +671 -0
  1308. tests/federation/test_federation_sender.py +956 -0
  1309. tests/federation/test_federation_server.py +631 -0
  1310. tests/federation/transport/__init__.py +0 -0
  1311. tests/federation/transport/server/__init__.py +20 -0
  1312. tests/federation/transport/server/test__base.py +156 -0
  1313. tests/federation/transport/test_client.py +157 -0
  1314. tests/federation/transport/test_knocking.py +323 -0
  1315. tests/federation/transport/test_server.py +74 -0
  1316. tests/handlers/__init__.py +0 -0
  1317. tests/handlers/oidc_test_key.p8 +5 -0
  1318. tests/handlers/oidc_test_key.pub.pem +4 -0
  1319. tests/handlers/test_admin.py +361 -0
  1320. tests/handlers/test_appservice.py +1336 -0
  1321. tests/handlers/test_auth.py +248 -0
  1322. tests/handlers/test_cas.py +239 -0
  1323. tests/handlers/test_deactivate_account.py +485 -0
  1324. tests/handlers/test_device.py +665 -0
  1325. tests/handlers/test_directory.py +613 -0
  1326. tests/handlers/test_e2e_keys.py +2025 -0
  1327. tests/handlers/test_e2e_room_keys.py +569 -0
  1328. tests/handlers/test_federation.py +794 -0
  1329. tests/handlers/test_federation_event.py +1181 -0
  1330. tests/handlers/test_message.py +322 -0
  1331. tests/handlers/test_oauth_delegation.py +1314 -0
  1332. tests/handlers/test_oidc.py +1688 -0
  1333. tests/handlers/test_password_providers.py +987 -0
  1334. tests/handlers/test_presence.py +2144 -0
  1335. tests/handlers/test_profile.py +401 -0
  1336. tests/handlers/test_receipts.py +342 -0
  1337. tests/handlers/test_register.py +880 -0
  1338. tests/handlers/test_room.py +108 -0
  1339. tests/handlers/test_room_list.py +93 -0
  1340. tests/handlers/test_room_member.py +764 -0
  1341. tests/handlers/test_room_policy.py +468 -0
  1342. tests/handlers/test_room_summary.py +1248 -0
  1343. tests/handlers/test_saml.py +427 -0
  1344. tests/handlers/test_send_email.py +230 -0
  1345. tests/handlers/test_sliding_sync.py +5065 -0
  1346. tests/handlers/test_sso.py +152 -0
  1347. tests/handlers/test_stats.py +594 -0
  1348. tests/handlers/test_sync.py +1275 -0
  1349. tests/handlers/test_typing.py +557 -0
  1350. tests/handlers/test_user_directory.py +1435 -0
  1351. tests/handlers/test_worker_lock.py +126 -0
  1352. tests/http/__init__.py +196 -0
  1353. tests/http/ca.crt +19 -0
  1354. tests/http/ca.key +27 -0
  1355. tests/http/federation/__init__.py +19 -0
  1356. tests/http/federation/test_matrix_federation_agent.py +1855 -0
  1357. tests/http/federation/test_srv_resolver.py +220 -0
  1358. tests/http/server/__init__.py +20 -0
  1359. tests/http/server/_base.py +621 -0
  1360. tests/http/server.key +27 -0
  1361. tests/http/test_additional_resource.py +76 -0
  1362. tests/http/test_client.py +422 -0
  1363. tests/http/test_endpoint.py +62 -0
  1364. tests/http/test_matrixfederationclient.py +1092 -0
  1365. tests/http/test_proxy.py +75 -0
  1366. tests/http/test_proxyagent.py +1008 -0
  1367. tests/http/test_servlet.py +145 -0
  1368. tests/http/test_simple_client.py +188 -0
  1369. tests/http/test_site.py +247 -0
  1370. tests/logging/__init__.py +42 -0
  1371. tests/logging/test_loggers.py +127 -0
  1372. tests/logging/test_opentracing.py +524 -0
  1373. tests/logging/test_remote_handler.py +184 -0
  1374. tests/logging/test_terse_json.py +253 -0
  1375. tests/media/__init__.py +20 -0
  1376. tests/media/test_base.py +88 -0
  1377. tests/media/test_filepath.py +602 -0
  1378. tests/media/test_html_preview.py +565 -0
  1379. tests/media/test_media_retention.py +299 -0
  1380. tests/media/test_media_storage.py +1401 -0
  1381. tests/media/test_oembed.py +172 -0
  1382. tests/media/test_url_previewer.py +120 -0
  1383. tests/metrics/__init__.py +0 -0
  1384. tests/metrics/test_background_process_metrics.py +21 -0
  1385. tests/metrics/test_metrics.py +407 -0
  1386. tests/metrics/test_phone_home_stats.py +263 -0
  1387. tests/module_api/__init__.py +0 -0
  1388. tests/module_api/test_account_data_manager.py +171 -0
  1389. tests/module_api/test_api.py +1035 -0
  1390. tests/module_api/test_event_unsigned_addition.py +66 -0
  1391. tests/module_api/test_spamchecker.py +286 -0
  1392. tests/push/__init__.py +0 -0
  1393. tests/push/test_bulk_push_rule_evaluator.py +652 -0
  1394. tests/push/test_email.py +570 -0
  1395. tests/push/test_http.py +1247 -0
  1396. tests/push/test_presentable_names.py +238 -0
  1397. tests/push/test_push_rule_evaluator.py +1069 -0
  1398. tests/replication/__init__.py +20 -0
  1399. tests/replication/_base.py +619 -0
  1400. tests/replication/http/__init__.py +20 -0
  1401. tests/replication/http/test__base.py +113 -0
  1402. tests/replication/storage/__init__.py +20 -0
  1403. tests/replication/storage/_base.py +85 -0
  1404. tests/replication/storage/test_events.py +299 -0
  1405. tests/replication/tcp/__init__.py +19 -0
  1406. tests/replication/tcp/streams/__init__.py +19 -0
  1407. tests/replication/tcp/streams/test_account_data.py +133 -0
  1408. tests/replication/tcp/streams/test_events.py +565 -0
  1409. tests/replication/tcp/streams/test_federation.py +117 -0
  1410. tests/replication/tcp/streams/test_partial_state.py +72 -0
  1411. tests/replication/tcp/streams/test_receipts.py +110 -0
  1412. tests/replication/tcp/streams/test_thread_subscriptions.py +157 -0
  1413. tests/replication/tcp/streams/test_to_device.py +112 -0
  1414. tests/replication/tcp/streams/test_typing.py +223 -0
  1415. tests/replication/tcp/test_commands.py +50 -0
  1416. tests/replication/tcp/test_handler.py +211 -0
  1417. tests/replication/test_auth.py +120 -0
  1418. tests/replication/test_client_reader_shard.py +101 -0
  1419. tests/replication/test_federation_ack.py +88 -0
  1420. tests/replication/test_federation_sender_shard.py +352 -0
  1421. tests/replication/test_module_cache_invalidation.py +89 -0
  1422. tests/replication/test_multi_media_repo.py +496 -0
  1423. tests/replication/test_pusher_shard.py +192 -0
  1424. tests/replication/test_sharded_event_persister.py +332 -0
  1425. tests/replication/test_sharded_receipts.py +250 -0
  1426. tests/rest/__init__.py +20 -0
  1427. tests/rest/admin/__init__.py +19 -0
  1428. tests/rest/admin/test_admin.py +614 -0
  1429. tests/rest/admin/test_background_updates.py +375 -0
  1430. tests/rest/admin/test_device.py +600 -0
  1431. tests/rest/admin/test_event.py +74 -0
  1432. tests/rest/admin/test_event_reports.py +781 -0
  1433. tests/rest/admin/test_federation.py +863 -0
  1434. tests/rest/admin/test_jwks.py +106 -0
  1435. tests/rest/admin/test_media.py +1091 -0
  1436. tests/rest/admin/test_registration_tokens.py +729 -0
  1437. tests/rest/admin/test_room.py +3626 -0
  1438. tests/rest/admin/test_scheduled_tasks.py +192 -0
  1439. tests/rest/admin/test_server_notice.py +753 -0
  1440. tests/rest/admin/test_statistics.py +523 -0
  1441. tests/rest/admin/test_user.py +6061 -0
  1442. tests/rest/admin/test_username_available.py +82 -0
  1443. tests/rest/client/__init__.py +20 -0
  1444. tests/rest/client/sliding_sync/__init__.py +13 -0
  1445. tests/rest/client/sliding_sync/test_connection_tracking.py +505 -0
  1446. tests/rest/client/sliding_sync/test_extension_account_data.py +1056 -0
  1447. tests/rest/client/sliding_sync/test_extension_e2ee.py +459 -0
  1448. tests/rest/client/sliding_sync/test_extension_receipts.py +934 -0
  1449. tests/rest/client/sliding_sync/test_extension_thread_subscriptions.py +497 -0
  1450. tests/rest/client/sliding_sync/test_extension_to_device.py +294 -0
  1451. tests/rest/client/sliding_sync/test_extension_typing.py +500 -0
  1452. tests/rest/client/sliding_sync/test_extensions.py +306 -0
  1453. tests/rest/client/sliding_sync/test_lists_filters.py +1975 -0
  1454. tests/rest/client/sliding_sync/test_room_subscriptions.py +303 -0
  1455. tests/rest/client/sliding_sync/test_rooms_invites.py +528 -0
  1456. tests/rest/client/sliding_sync/test_rooms_meta.py +1338 -0
  1457. tests/rest/client/sliding_sync/test_rooms_required_state.py +2247 -0
  1458. tests/rest/client/sliding_sync/test_rooms_timeline.py +718 -0
  1459. tests/rest/client/sliding_sync/test_sliding_sync.py +1688 -0
  1460. tests/rest/client/test_account.py +1543 -0
  1461. tests/rest/client/test_account_data.py +81 -0
  1462. tests/rest/client/test_auth.py +1508 -0
  1463. tests/rest/client/test_auth_metadata.py +145 -0
  1464. tests/rest/client/test_capabilities.py +318 -0
  1465. tests/rest/client/test_consent.py +138 -0
  1466. tests/rest/client/test_delayed_events.py +553 -0
  1467. tests/rest/client/test_devices.py +634 -0
  1468. tests/rest/client/test_directory.py +249 -0
  1469. tests/rest/client/test_ephemeral_message.py +113 -0
  1470. tests/rest/client/test_events.py +165 -0
  1471. tests/rest/client/test_filter.py +124 -0
  1472. tests/rest/client/test_identity.py +67 -0
  1473. tests/rest/client/test_keys.py +516 -0
  1474. tests/rest/client/test_login.py +1881 -0
  1475. tests/rest/client/test_login_token_request.py +175 -0
  1476. tests/rest/client/test_matrixrtc.py +105 -0
  1477. tests/rest/client/test_media.py +3156 -0
  1478. tests/rest/client/test_models.py +83 -0
  1479. tests/rest/client/test_mutual_rooms.py +235 -0
  1480. tests/rest/client/test_notifications.py +231 -0
  1481. tests/rest/client/test_owned_state.py +308 -0
  1482. tests/rest/client/test_password_policy.py +186 -0
  1483. tests/rest/client/test_power_levels.py +295 -0
  1484. tests/rest/client/test_presence.py +149 -0
  1485. tests/rest/client/test_profile.py +925 -0
  1486. tests/rest/client/test_push_rule_attrs.py +510 -0
  1487. tests/rest/client/test_read_marker.py +151 -0
  1488. tests/rest/client/test_receipts.py +287 -0
  1489. tests/rest/client/test_redactions.py +657 -0
  1490. tests/rest/client/test_register.py +1314 -0
  1491. tests/rest/client/test_relations.py +1954 -0
  1492. tests/rest/client/test_rendezvous.py +468 -0
  1493. tests/rest/client/test_reporting.py +324 -0
  1494. tests/rest/client/test_retention.py +389 -0
  1495. tests/rest/client/test_rooms.py +5486 -0
  1496. tests/rest/client/test_sendtodevice.py +271 -0
  1497. tests/rest/client/test_shadow_banned.py +335 -0
  1498. tests/rest/client/test_sync.py +1147 -0
  1499. tests/rest/client/test_tags.py +161 -0
  1500. tests/rest/client/test_third_party_rules.py +1076 -0
  1501. tests/rest/client/test_thread_subscriptions.py +351 -0
  1502. tests/rest/client/test_transactions.py +204 -0
  1503. tests/rest/client/test_typing.py +114 -0
  1504. tests/rest/client/test_upgrade_room.py +433 -0
  1505. tests/rest/client/utils.py +985 -0
  1506. tests/rest/key/__init__.py +0 -0
  1507. tests/rest/key/v2/__init__.py +0 -0
  1508. tests/rest/key/v2/test_remote_key_resource.py +282 -0
  1509. tests/rest/media/__init__.py +19 -0
  1510. tests/rest/media/test_domain_blocking.py +148 -0
  1511. tests/rest/media/test_url_preview.py +1445 -0
  1512. tests/rest/synapse/__init__.py +12 -0
  1513. tests/rest/synapse/client/__init__.py +12 -0
  1514. tests/rest/synapse/client/test_federation_whitelist.py +118 -0
  1515. tests/rest/synapse/mas/__init__.py +12 -0
  1516. tests/rest/synapse/mas/_base.py +43 -0
  1517. tests/rest/synapse/mas/test_devices.py +693 -0
  1518. tests/rest/synapse/mas/test_users.py +1399 -0
  1519. tests/rest/test_health.py +35 -0
  1520. tests/rest/test_well_known.py +155 -0
  1521. tests/scripts/__init__.py +0 -0
  1522. tests/scripts/test_new_matrix_user.py +172 -0
  1523. tests/server.py +1374 -0
  1524. tests/server_notices/__init__.py +241 -0
  1525. tests/server_notices/test_consent.py +111 -0
  1526. tests/server_notices/test_resource_limits_server_notices.py +409 -0
  1527. tests/state/__init__.py +0 -0
  1528. tests/state/test_v2.py +1096 -0
  1529. tests/state/test_v21.py +506 -0
  1530. tests/storage/__init__.py +0 -0
  1531. tests/storage/databases/__init__.py +20 -0
  1532. tests/storage/databases/main/__init__.py +20 -0
  1533. tests/storage/databases/main/test_cache.py +124 -0
  1534. tests/storage/databases/main/test_deviceinbox.py +323 -0
  1535. tests/storage/databases/main/test_end_to_end_keys.py +127 -0
  1536. tests/storage/databases/main/test_events_worker.py +594 -0
  1537. tests/storage/databases/main/test_lock.py +499 -0
  1538. tests/storage/databases/main/test_metrics.py +88 -0
  1539. tests/storage/databases/main/test_receipts.py +218 -0
  1540. tests/storage/databases/main/test_room.py +192 -0
  1541. tests/storage/test__base.py +178 -0
  1542. tests/storage/test_account_data.py +186 -0
  1543. tests/storage/test_appservice.py +568 -0
  1544. tests/storage/test_background_update.py +671 -0
  1545. tests/storage/test_base.py +813 -0
  1546. tests/storage/test_cleanup_extrems.py +396 -0
  1547. tests/storage/test_client_ips.py +788 -0
  1548. tests/storage/test_database.py +288 -0
  1549. tests/storage/test_devices.py +353 -0
  1550. tests/storage/test_directory.py +74 -0
  1551. tests/storage/test_e2e_room_keys.py +87 -0
  1552. tests/storage/test_end_to_end_keys.py +120 -0
  1553. tests/storage/test_event_chain.py +826 -0
  1554. tests/storage/test_event_federation.py +1433 -0
  1555. tests/storage/test_event_push_actions.py +809 -0
  1556. tests/storage/test_events.py +591 -0
  1557. tests/storage/test_events_bg_updates.py +156 -0
  1558. tests/storage/test_id_generators.py +791 -0
  1559. tests/storage/test_invite_rule.py +171 -0
  1560. tests/storage/test_main.py +56 -0
  1561. tests/storage/test_monthly_active_users.py +500 -0
  1562. tests/storage/test_profile.py +134 -0
  1563. tests/storage/test_purge.py +459 -0
  1564. tests/storage/test_receipts.py +309 -0
  1565. tests/storage/test_redaction.py +462 -0
  1566. tests/storage/test_registration.py +277 -0
  1567. tests/storage/test_relations.py +118 -0
  1568. tests/storage/test_rollback_worker.py +132 -0
  1569. tests/storage/test_room.py +69 -0
  1570. tests/storage/test_room_search.py +383 -0
  1571. tests/storage/test_roommember.py +812 -0
  1572. tests/storage/test_sliding_sync_tables.py +5187 -0
  1573. tests/storage/test_state.py +959 -0
  1574. tests/storage/test_state_deletion.py +475 -0
  1575. tests/storage/test_stream.py +1533 -0
  1576. tests/storage/test_thread_subscriptions.py +369 -0
  1577. tests/storage/test_transactions.py +77 -0
  1578. tests/storage/test_txn_limit.py +49 -0
  1579. tests/storage/test_unsafe_locale.py +67 -0
  1580. tests/storage/test_user_directory.py +691 -0
  1581. tests/storage/test_user_filters.py +101 -0
  1582. tests/storage/util/__init__.py +20 -0
  1583. tests/storage/util/test_partial_state_events_tracker.py +181 -0
  1584. tests/synapse_rust/__init__.py +11 -0
  1585. tests/synapse_rust/test_http_client.py +225 -0
  1586. tests/test_distributor.py +74 -0
  1587. tests/test_event_auth.py +921 -0
  1588. tests/test_mau.py +347 -0
  1589. tests/test_phone_home.py +102 -0
  1590. tests/test_rust.py +11 -0
  1591. tests/test_server.py +557 -0
  1592. tests/test_state.py +902 -0
  1593. tests/test_terms_auth.py +128 -0
  1594. tests/test_types.py +201 -0
  1595. tests/test_utils/__init__.py +161 -0
  1596. tests/test_utils/event_injection.py +150 -0
  1597. tests/test_utils/html_parsers.py +59 -0
  1598. tests/test_utils/logging_setup.py +74 -0
  1599. tests/test_utils/oidc.py +370 -0
  1600. tests/test_visibility.py +712 -0
  1601. tests/types/__init__.py +0 -0
  1602. tests/types/test_init.py +51 -0
  1603. tests/types/test_state.py +627 -0
  1604. tests/unittest.py +1108 -0
  1605. tests/util/__init__.py +20 -0
  1606. tests/util/caches/__init__.py +20 -0
  1607. tests/util/caches/test_cached_call.py +168 -0
  1608. tests/util/caches/test_deferred_cache.py +317 -0
  1609. tests/util/caches/test_descriptors.py +1110 -0
  1610. tests/util/caches/test_response_cache.py +225 -0
  1611. tests/util/caches/test_ttlcache.py +90 -0
  1612. tests/util/test_async_helpers.py +808 -0
  1613. tests/util/test_background_queue.py +117 -0
  1614. tests/util/test_batching_queue.py +252 -0
  1615. tests/util/test_check_dependencies.py +243 -0
  1616. tests/util/test_dict_cache.py +130 -0
  1617. tests/util/test_events.py +118 -0
  1618. tests/util/test_expiring_cache.py +113 -0
  1619. tests/util/test_file_consumer.py +199 -0
  1620. tests/util/test_itertools.py +190 -0
  1621. tests/util/test_linearizer.py +264 -0
  1622. tests/util/test_logcontext.py +715 -0
  1623. tests/util/test_logformatter.py +44 -0
  1624. tests/util/test_lrucache.py +479 -0
  1625. tests/util/test_macaroons.py +126 -0
  1626. tests/util/test_mutable_overlay_mapping.py +189 -0
  1627. tests/util/test_ratelimitutils.py +146 -0
  1628. tests/util/test_retryutils.py +314 -0
  1629. tests/util/test_rwlock.py +401 -0
  1630. tests/util/test_stream_change_cache.py +304 -0
  1631. tests/util/test_stringutils.py +86 -0
  1632. tests/util/test_task_scheduler.py +227 -0
  1633. tests/util/test_threepids.py +55 -0
  1634. tests/util/test_treecache.py +93 -0
  1635. tests/util/test_wheel_timer.py +82 -0
  1636. tests/utils.py +342 -0
tests/rest/media/test_url_preview.py ADDED
@@ -0,0 +1,1445 @@
1
+ #
2
+ # This file is licensed under the Affero General Public License (AGPL) version 3.
3
+ #
4
+ # Copyright 2021 The Matrix.org Foundation C.I.C.
5
+ # Copyright (C) 2023 New Vector, Ltd
6
+ #
7
+ # This program is free software: you can redistribute it and/or modify
8
+ # it under the terms of the GNU Affero General Public License as
9
+ # published by the Free Software Foundation, either version 3 of the
10
+ # License, or (at your option) any later version.
11
+ #
12
+ # See the GNU Affero General Public License for more details:
13
+ # <https://www.gnu.org/licenses/agpl-3.0.html>.
14
+ #
15
+ # Originally licensed under the Apache License, Version 2.0:
16
+ # <http://www.apache.org/licenses/LICENSE-2.0>.
17
+ #
18
+ # [This file includes modifications made by New Vector Limited]
19
+ #
20
+ #
21
+ import base64
22
+ import json
23
+ import os
24
+ import re
25
+ from typing import Any, Sequence
26
+ from urllib.parse import quote, urlencode
27
+
28
+ from twisted.internet._resolver import HostResolution
29
+ from twisted.internet.address import IPv4Address, IPv6Address
30
+ from twisted.internet.error import DNSLookupError
31
+ from twisted.internet.interfaces import IAddress, IResolutionReceiver
32
+ from twisted.internet.testing import AccumulatingProtocol, MemoryReactor
33
+ from twisted.web.resource import Resource
34
+
35
+ from synapse.config.oembed import OEmbedEndpointConfig
36
+ from synapse.media.url_previewer import IMAGE_CACHE_EXPIRY_MS
37
+ from synapse.server import HomeServer
38
+ from synapse.types import JsonDict
39
+ from synapse.util.clock import Clock
40
+ from synapse.util.stringutils import parse_and_validate_mxc_uri
41
+
42
+ from tests import unittest
43
+ from tests.server import FakeTransport
44
+ from tests.test_utils import SMALL_PNG
45
+ from tests.unittest import override_config
46
+
47
+ try:
48
+ import lxml
49
+ except ImportError:
50
+ lxml = None # type: ignore[assignment]
51
+
52
+
53
+ class URLPreviewTests(unittest.HomeserverTestCase):
54
+ if not lxml:
55
+ skip = "url preview feature requires lxml"
56
+
57
+ hijack_auth = True
58
+ user_id = "@test:user"
59
+ end_content = (
60
+ b"<html><head>"
61
+ b'<meta property="og:title" content="~matrix~" />'
62
+ b'<meta property="og:description" content="hi" />'
63
+ b"</head></html>"
64
+ )
65
+
66
+ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
67
+ config = self.default_config()
68
+ config["url_preview_enabled"] = True
69
+ config["max_spider_size"] = 9999999
70
+ config["url_preview_ip_range_blacklist"] = (
71
+ "192.168.1.1",
72
+ "1.0.0.0/8",
73
+ "3fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
74
+ "2001:800::/21",
75
+ )
76
+ config["url_preview_ip_range_whitelist"] = ("1.1.1.1",)
77
+ config["url_preview_accept_language"] = [
78
+ "en-UK",
79
+ "en-US;q=0.9",
80
+ "fr;q=0.8",
81
+ "*;q=0.7",
82
+ ]
83
+
84
+ self.storage_path = self.mktemp()
85
+ self.media_store_path = self.mktemp()
86
+ os.mkdir(self.storage_path)
87
+ os.mkdir(self.media_store_path)
88
+ config["media_store_path"] = self.media_store_path
89
+
90
+ provider_config = {
91
+ "module": "synapse.media.storage_provider.FileStorageProviderBackend",
92
+ "store_local": True,
93
+ "store_synchronous": False,
94
+ "store_remote": True,
95
+ "config": {"directory": self.storage_path},
96
+ }
97
+
98
+ config["media_storage_providers"] = [provider_config]
99
+
100
+ hs = self.setup_test_homeserver(config=config)
101
+
102
+ # After the hs is created, modify the parsed oEmbed config (to avoid
103
+ # messing with files).
104
+ #
105
+ # Note that HTTP URLs are used to avoid having to deal with TLS in tests.
106
+ hs.config.oembed.oembed_patterns = [
107
+ OEmbedEndpointConfig(
108
+ api_endpoint="http://publish.twitter.com/oembed",
109
+ url_patterns=[
110
+ re.compile(r"http://twitter\.com/.+/status/.+"),
111
+ ],
112
+ formats=None,
113
+ ),
114
+ OEmbedEndpointConfig(
115
+ api_endpoint="http://www.hulu.com/api/oembed.{format}",
116
+ url_patterns=[
117
+ re.compile(r"http://www\.hulu\.com/watch/.+"),
118
+ ],
119
+ formats=["json"],
120
+ ),
121
+ ]
122
+
123
+ return hs
124
+
125
+ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
126
+ self.media_repo = hs.get_media_repository()
127
+ assert self.media_repo.url_previewer is not None
128
+ self.url_previewer = self.media_repo.url_previewer
129
+
130
+ self.lookups: dict[str, Any] = {}
131
+
132
+ class Resolver:
133
+ def resolveHostName(
134
+ _self,
135
+ resolutionReceiver: IResolutionReceiver,
136
+ hostName: str,
137
+ portNumber: int = 0,
138
+ addressTypes: Sequence[type[IAddress]] | None = None,
139
+ transportSemantics: str = "TCP",
140
+ ) -> IResolutionReceiver:
141
+ resolution = HostResolution(hostName)
142
+ resolutionReceiver.resolutionBegan(resolution)
143
+ if hostName not in self.lookups:
144
+ raise DNSLookupError("OH NO")
145
+
146
+ for i in self.lookups[hostName]:
147
+ resolutionReceiver.addressResolved(i[0]("TCP", i[1], portNumber))
148
+ resolutionReceiver.resolutionComplete()
149
+ return resolutionReceiver
150
+
151
+ self.reactor.nameResolver = Resolver() # type: ignore[assignment]
152
+
153
+ def create_resource_dict(self) -> dict[str, Resource]:
154
+ """Create a resource tree for the test server
155
+
156
+ A resource tree is a mapping from path to twisted.web.resource.
157
+
158
+ The default implementation creates a JsonResource and calls each function in
159
+ `servlets` to register servlets against it.
160
+ """
161
+ return {"/_matrix/media": self.hs.get_media_repository_resource()}
162
+
163
+ def _assert_small_png(self, json_body: JsonDict) -> None:
164
+ """Assert properties from the SMALL_PNG test image."""
165
+ self.assertTrue(json_body["og:image"].startswith("mxc://"))
166
+ self.assertEqual(json_body["og:image:height"], 1)
167
+ self.assertEqual(json_body["og:image:width"], 1)
168
+ self.assertEqual(json_body["og:image:type"], "image/png")
169
+ self.assertEqual(json_body["matrix:image:size"], 67)
170
+
171
+ def test_cache_returns_correct_type(self) -> None:
172
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
173
+
174
+ channel = self.make_request(
175
+ "GET",
176
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
177
+ shorthand=False,
178
+ await_result=False,
179
+ )
180
+ self.pump()
181
+
182
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
183
+ server = AccumulatingProtocol()
184
+ server.makeConnection(FakeTransport(client, self.reactor))
185
+ client.makeConnection(FakeTransport(server, self.reactor))
186
+ client.dataReceived(
187
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
188
+ % (len(self.end_content),)
189
+ + self.end_content
190
+ )
191
+
192
+ self.pump()
193
+ self.assertEqual(channel.code, 200)
194
+ self.assertEqual(
195
+ channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
196
+ )
197
+
198
+ # Check the cache returns the correct response
199
+ channel = self.make_request(
200
+ "GET",
201
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
202
+ shorthand=False,
203
+ )
204
+
205
+ # Check the cache response has the same content
206
+ self.assertEqual(channel.code, 200)
207
+ self.assertEqual(
208
+ channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
209
+ )
210
+
211
+ # Clear the in-memory cache
212
+ self.assertIn("http://matrix.org", self.url_previewer._cache)
213
+ self.url_previewer._cache.pop("http://matrix.org")
214
+ self.assertNotIn("http://matrix.org", self.url_previewer._cache)
215
+
216
+ # Check the database cache returns the correct response
217
+ channel = self.make_request(
218
+ "GET",
219
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
220
+ shorthand=False,
221
+ )
222
+
223
+ # Check the cache response has the same content
224
+ self.assertEqual(channel.code, 200)
225
+ self.assertEqual(
226
+ channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
227
+ )
228
+
229
+ def test_non_ascii_preview_httpequiv(self) -> None:
230
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
231
+
232
+ end_content = (
233
+ b"<html><head>"
234
+ b'<meta http-equiv="Content-Type" content="text/html; charset=windows-1251"/>'
235
+ b'<meta property="og:title" content="\xe4\xea\xe0" />'
236
+ b'<meta property="og:description" content="hi" />'
237
+ b"</head></html>"
238
+ )
239
+
240
+ channel = self.make_request(
241
+ "GET",
242
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
243
+ shorthand=False,
244
+ await_result=False,
245
+ )
246
+ self.pump()
247
+
248
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
249
+ server = AccumulatingProtocol()
250
+ server.makeConnection(FakeTransport(client, self.reactor))
251
+ client.makeConnection(FakeTransport(server, self.reactor))
252
+ client.dataReceived(
253
+ (
254
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
255
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
256
+ )
257
+ % (len(end_content),)
258
+ + end_content
259
+ )
260
+
261
+ self.pump()
262
+ self.assertEqual(channel.code, 200)
263
+ self.assertEqual(channel.json_body["og:title"], "\u0434\u043a\u0430")
264
+
265
+ def test_video_rejected(self) -> None:
266
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
267
+
268
+ end_content = b"anything"
269
+
270
+ channel = self.make_request(
271
+ "GET",
272
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
273
+ shorthand=False,
274
+ await_result=False,
275
+ )
276
+ self.pump()
277
+
278
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
279
+ server = AccumulatingProtocol()
280
+ server.makeConnection(FakeTransport(client, self.reactor))
281
+ client.makeConnection(FakeTransport(server, self.reactor))
282
+ client.dataReceived(
283
+ (
284
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
285
+ b"Content-Type: video/mp4\r\n\r\n"
286
+ )
287
+ % (len(end_content))
288
+ + end_content
289
+ )
290
+
291
+ self.pump()
292
+ self.assertEqual(channel.code, 502)
293
+ self.assertEqual(
294
+ channel.json_body,
295
+ {
296
+ "errcode": "M_UNKNOWN",
297
+ "error": "Requested file's content type not allowed for this operation: video/mp4",
298
+ },
299
+ )
300
+
301
+ def test_audio_rejected(self) -> None:
302
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
303
+
304
+ end_content = b"anything"
305
+
306
+ channel = self.make_request(
307
+ "GET",
308
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
309
+ shorthand=False,
310
+ await_result=False,
311
+ )
312
+ self.pump()
313
+
314
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
315
+ server = AccumulatingProtocol()
316
+ server.makeConnection(FakeTransport(client, self.reactor))
317
+ client.makeConnection(FakeTransport(server, self.reactor))
318
+ client.dataReceived(
319
+ (
320
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
321
+ b"Content-Type: audio/aac\r\n\r\n"
322
+ )
323
+ % (len(end_content))
324
+ + end_content
325
+ )
326
+
327
+ self.pump()
328
+ self.assertEqual(channel.code, 502)
329
+ self.assertEqual(
330
+ channel.json_body,
331
+ {
332
+ "errcode": "M_UNKNOWN",
333
+ "error": "Requested file's content type not allowed for this operation: audio/aac",
334
+ },
335
+ )
336
+
337
+ def test_non_ascii_preview_content_type(self) -> None:
338
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
339
+
340
+ end_content = (
341
+ b"<html><head>"
342
+ b'<meta property="og:title" content="\xe4\xea\xe0" />'
343
+ b'<meta property="og:description" content="hi" />'
344
+ b"</head></html>"
345
+ )
346
+
347
+ channel = self.make_request(
348
+ "GET",
349
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
350
+ shorthand=False,
351
+ await_result=False,
352
+ )
353
+ self.pump()
354
+
355
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
356
+ server = AccumulatingProtocol()
357
+ server.makeConnection(FakeTransport(client, self.reactor))
358
+ client.makeConnection(FakeTransport(server, self.reactor))
359
+ client.dataReceived(
360
+ (
361
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
362
+ b'Content-Type: text/html; charset="windows-1251"\r\n\r\n'
363
+ )
364
+ % (len(end_content),)
365
+ + end_content
366
+ )
367
+
368
+ self.pump()
369
+ self.assertEqual(channel.code, 200)
370
+ self.assertEqual(channel.json_body["og:title"], "\u0434\u043a\u0430")
371
+
372
+ def test_overlong_title(self) -> None:
373
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
374
+
375
+ end_content = (
376
+ b"<html><head>"
377
+ b"<title>" + b"x" * 2000 + b"</title>"
378
+ b'<meta property="og:description" content="hi" />'
379
+ b"</head></html>"
380
+ )
381
+
382
+ channel = self.make_request(
383
+ "GET",
384
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
385
+ shorthand=False,
386
+ await_result=False,
387
+ )
388
+ self.pump()
389
+
390
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
391
+ server = AccumulatingProtocol()
392
+ server.makeConnection(FakeTransport(client, self.reactor))
393
+ client.makeConnection(FakeTransport(server, self.reactor))
394
+ client.dataReceived(
395
+ (
396
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
397
+ b'Content-Type: text/html; charset="windows-1251"\r\n\r\n'
398
+ )
399
+ % (len(end_content),)
400
+ + end_content
401
+ )
402
+
403
+ self.pump()
404
+ self.assertEqual(channel.code, 200)
405
+ res = channel.json_body
406
+ # We should only see the `og:description` field, as `title` is too long and should be stripped out
407
+ self.assertCountEqual(["og:description"], res.keys())
408
+
409
+ def test_ipaddr(self) -> None:
410
+ """
411
+ IP addresses can be previewed directly.
412
+ """
413
+ self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")]
414
+
415
+ channel = self.make_request(
416
+ "GET",
417
+ "/_matrix/media/v3/preview_url?url=http://example.com",
418
+ shorthand=False,
419
+ await_result=False,
420
+ )
421
+ self.pump()
422
+
423
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
424
+ server = AccumulatingProtocol()
425
+ server.makeConnection(FakeTransport(client, self.reactor))
426
+ client.makeConnection(FakeTransport(server, self.reactor))
427
+ client.dataReceived(
428
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
429
+ % (len(self.end_content),)
430
+ + self.end_content
431
+ )
432
+
433
+ self.pump()
434
+ self.assertEqual(channel.code, 200)
435
+ self.assertEqual(
436
+ channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
437
+ )
438
+
439
+ def test_blocked_ip_specific(self) -> None:
440
+ """
441
+ Blocked IP addresses, found via DNS, are not spidered.
442
+ """
443
+ self.lookups["example.com"] = [(IPv4Address, "192.168.1.1")]
444
+
445
+ channel = self.make_request(
446
+ "GET",
447
+ "/_matrix/media/v3/preview_url?url=http://example.com",
448
+ shorthand=False,
449
+ )
450
+
451
+ # No requests made.
452
+ self.assertEqual(len(self.reactor.tcpClients), 0)
453
+ self.assertEqual(channel.code, 502)
454
+ self.assertEqual(
455
+ channel.json_body,
456
+ {
457
+ "errcode": "M_UNKNOWN",
458
+ "error": "DNS resolution failure during URL preview generation",
459
+ },
460
+ )
461
+
462
+ def test_blocked_ip_range(self) -> None:
463
+ """
464
+ Blocked IP ranges, IPs found over DNS, are not spidered.
465
+ """
466
+ self.lookups["example.com"] = [(IPv4Address, "1.1.1.2")]
467
+
468
+ channel = self.make_request(
469
+ "GET",
470
+ "/_matrix/media/v3/preview_url?url=http://example.com",
471
+ shorthand=False,
472
+ )
473
+
474
+ self.assertEqual(channel.code, 502)
475
+ self.assertEqual(
476
+ channel.json_body,
477
+ {
478
+ "errcode": "M_UNKNOWN",
479
+ "error": "DNS resolution failure during URL preview generation",
480
+ },
481
+ )
482
+
483
+ def test_blocked_ip_specific_direct(self) -> None:
484
+ """
485
+ Blocked IP addresses, accessed directly, are not spidered.
486
+ """
487
+ channel = self.make_request(
488
+ "GET",
489
+ "/_matrix/media/v3/preview_url?url=http://192.168.1.1",
490
+ shorthand=False,
491
+ )
492
+
493
+ # No requests made.
494
+ self.assertEqual(len(self.reactor.tcpClients), 0)
495
+ self.assertEqual(
496
+ channel.json_body,
497
+ {"errcode": "M_UNKNOWN", "error": "IP address blocked"},
498
+ )
499
+ self.assertEqual(channel.code, 403)
500
+
501
+ def test_blocked_ip_range_direct(self) -> None:
502
+ """
503
+ Blocked IP ranges, accessed directly, are not spidered.
504
+ """
505
+ channel = self.make_request(
506
+ "GET", "/_matrix/media/v3/preview_url?url=http://1.1.1.2", shorthand=False
507
+ )
508
+
509
+ self.assertEqual(channel.code, 403)
510
+ self.assertEqual(
511
+ channel.json_body,
512
+ {"errcode": "M_UNKNOWN", "error": "IP address blocked"},
513
+ )
514
+
515
+ def test_blocked_ip_range_whitelisted_ip(self) -> None:
516
+ """
517
+ Blocked but then subsequently whitelisted IP addresses can be
518
+ spidered.
519
+ """
520
+ self.lookups["example.com"] = [(IPv4Address, "1.1.1.1")]
521
+
522
+ channel = self.make_request(
523
+ "GET",
524
+ "/_matrix/media/v3/preview_url?url=http://example.com",
525
+ shorthand=False,
526
+ await_result=False,
527
+ )
528
+ self.pump()
529
+
530
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
531
+
532
+ server = AccumulatingProtocol()
533
+ server.makeConnection(FakeTransport(client, self.reactor))
534
+ client.makeConnection(FakeTransport(server, self.reactor))
535
+
536
+ client.dataReceived(
537
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
538
+ % (len(self.end_content),)
539
+ + self.end_content
540
+ )
541
+
542
+ self.pump()
543
+ self.assertEqual(channel.code, 200)
544
+ self.assertEqual(
545
+ channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
546
+ )
547
+
548
+ def test_blocked_ip_with_external_ip(self) -> None:
549
+ """
550
+ If a hostname resolves a blocked IP, even if there's a non-blocked one,
551
+ it will be rejected.
552
+ """
553
+ # Hardcode the URL resolving to the IP we want.
554
+ self.lookups["example.com"] = [
555
+ (IPv4Address, "1.1.1.2"),
556
+ (IPv4Address, "10.1.2.3"),
557
+ ]
558
+
559
+ channel = self.make_request(
560
+ "GET",
561
+ "/_matrix/media/v3/preview_url?url=http://example.com",
562
+ shorthand=False,
563
+ )
564
+ self.assertEqual(channel.code, 502)
565
+ self.assertEqual(
566
+ channel.json_body,
567
+ {
568
+ "errcode": "M_UNKNOWN",
569
+ "error": "DNS resolution failure during URL preview generation",
570
+ },
571
+ )
572
+
573
+ def test_blocked_ipv6_specific(self) -> None:
574
+ """
575
+ Blocked IP addresses, found via DNS, are not spidered.
576
+ """
577
+ self.lookups["example.com"] = [
578
+ (IPv6Address, "3fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")
579
+ ]
580
+
581
+ channel = self.make_request(
582
+ "GET",
583
+ "/_matrix/media/v3/preview_url?url=http://example.com",
584
+ shorthand=False,
585
+ )
586
+
587
+ # No requests made.
588
+ self.assertEqual(len(self.reactor.tcpClients), 0)
589
+ self.assertEqual(channel.code, 502)
590
+ self.assertEqual(
591
+ channel.json_body,
592
+ {
593
+ "errcode": "M_UNKNOWN",
594
+ "error": "DNS resolution failure during URL preview generation",
595
+ },
596
+ )
597
+
598
+ def test_blocked_ipv6_range(self) -> None:
599
+ """
600
+ Blocked IP ranges, IPs found over DNS, are not spidered.
601
+ """
602
+ self.lookups["example.com"] = [(IPv6Address, "2001:800::1")]
603
+
604
+ channel = self.make_request(
605
+ "GET",
606
+ "/_matrix/media/v3/preview_url?url=http://example.com",
607
+ shorthand=False,
608
+ )
609
+
610
+ self.assertEqual(channel.code, 502)
611
+ self.assertEqual(
612
+ channel.json_body,
613
+ {
614
+ "errcode": "M_UNKNOWN",
615
+ "error": "DNS resolution failure during URL preview generation",
616
+ },
617
+ )
618
+
619
+ def test_OPTIONS(self) -> None:
620
+ """
621
+ OPTIONS returns the OPTIONS.
622
+ """
623
+ channel = self.make_request(
624
+ "OPTIONS",
625
+ "/_matrix/media/v3/preview_url?url=http://example.com",
626
+ shorthand=False,
627
+ )
628
+ self.assertEqual(channel.code, 204)
629
+
630
+ def test_accept_language_config_option(self) -> None:
631
+ """
632
+ Accept-Language header is sent to the remote server
633
+ """
634
+ self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")]
635
+
636
+ # Build and make a request to the server
637
+ channel = self.make_request(
638
+ "GET",
639
+ "/_matrix/media/v3/preview_url?url=http://example.com",
640
+ shorthand=False,
641
+ await_result=False,
642
+ )
643
+ self.pump()
644
+
645
+ # Extract Synapse's tcp client
646
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
647
+
648
+ # Build a fake remote server to reply with
649
+ server = AccumulatingProtocol()
650
+
651
+ # Connect the two together
652
+ server.makeConnection(FakeTransport(client, self.reactor))
653
+ client.makeConnection(FakeTransport(server, self.reactor))
654
+
655
+ # Tell Synapse that it has received some data from the remote server
656
+ client.dataReceived(
657
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
658
+ % (len(self.end_content),)
659
+ + self.end_content
660
+ )
661
+
662
+ # Move the reactor along until we get a response on our original channel
663
+ self.pump()
664
+ self.assertEqual(channel.code, 200)
665
+ self.assertEqual(
666
+ channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
667
+ )
668
+
669
+ # Check that the server received the Accept-Language header as part
670
+ # of the request from Synapse
671
+ self.assertIn(
672
+ (
673
+ b"Accept-Language: en-UK\r\n"
674
+ b"Accept-Language: en-US;q=0.9\r\n"
675
+ b"Accept-Language: fr;q=0.8\r\n"
676
+ b"Accept-Language: *;q=0.7"
677
+ ),
678
+ server.data,
679
+ )
680
+
681
+ def test_image(self) -> None:
682
+ """An image should be precached if mentioned in the HTML."""
683
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
684
+ self.lookups["cdn.matrix.org"] = [(IPv4Address, "10.1.2.4")]
685
+
686
+ result = (
687
+ b"""<html><body><img src="http://cdn.matrix.org/foo.png"></body></html>"""
688
+ )
689
+
690
+ channel = self.make_request(
691
+ "GET",
692
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
693
+ shorthand=False,
694
+ await_result=False,
695
+ )
696
+ self.pump()
697
+
698
+ # Respond with the HTML.
699
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
700
+ server = AccumulatingProtocol()
701
+ server.makeConnection(FakeTransport(client, self.reactor))
702
+ client.makeConnection(FakeTransport(server, self.reactor))
703
+ client.dataReceived(
704
+ (
705
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
706
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
707
+ )
708
+ % (len(result),)
709
+ + result
710
+ )
711
+ self.pump()
712
+
713
+ # Respond with the photo.
714
+ client = self.reactor.tcpClients[1][2].buildProtocol(None)
715
+ server = AccumulatingProtocol()
716
+ server.makeConnection(FakeTransport(client, self.reactor))
717
+ client.makeConnection(FakeTransport(server, self.reactor))
718
+ client.dataReceived(
719
+ (
720
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
721
+ b"Content-Type: image/png\r\n\r\n"
722
+ )
723
+ % (len(SMALL_PNG),)
724
+ + SMALL_PNG
725
+ )
726
+ self.pump()
727
+
728
+ # The image should be in the result.
729
+ self.assertEqual(channel.code, 200)
730
+ self._assert_small_png(channel.json_body)
731
+
732
+ def test_nonexistent_image(self) -> None:
733
+ """If the preview image doesn't exist, ensure some data is returned."""
734
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
735
+
736
+ result = (
737
+ b"""<html><body><img src="http://cdn.matrix.org/foo.jpg"></body></html>"""
738
+ )
739
+
740
+ channel = self.make_request(
741
+ "GET",
742
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
743
+ shorthand=False,
744
+ await_result=False,
745
+ )
746
+ self.pump()
747
+
748
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
749
+ server = AccumulatingProtocol()
750
+ server.makeConnection(FakeTransport(client, self.reactor))
751
+ client.makeConnection(FakeTransport(server, self.reactor))
752
+ client.dataReceived(
753
+ (
754
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
755
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
756
+ )
757
+ % (len(result),)
758
+ + result
759
+ )
760
+
761
+ self.pump()
762
+
763
+ # There should not be a second connection.
764
+ self.assertEqual(len(self.reactor.tcpClients), 1)
765
+
766
+ # The image should not be in the result.
767
+ self.assertEqual(channel.code, 200)
768
+ self.assertNotIn("og:image", channel.json_body)
769
+
770
+ @unittest.override_config(
771
+ {"url_preview_url_blacklist": [{"netloc": "cdn.matrix.org"}]}
772
+ )
773
+ def test_image_blocked(self) -> None:
774
+ """If the preview image doesn't exist, ensure some data is returned."""
775
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
776
+ self.lookups["cdn.matrix.org"] = [(IPv4Address, "10.1.2.4")]
777
+
778
+ result = (
779
+ b"""<html><body><img src="http://cdn.matrix.org/foo.jpg"></body></html>"""
780
+ )
781
+
782
+ channel = self.make_request(
783
+ "GET",
784
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
785
+ shorthand=False,
786
+ await_result=False,
787
+ )
788
+ self.pump()
789
+
790
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
791
+ server = AccumulatingProtocol()
792
+ server.makeConnection(FakeTransport(client, self.reactor))
793
+ client.makeConnection(FakeTransport(server, self.reactor))
794
+ client.dataReceived(
795
+ (
796
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
797
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
798
+ )
799
+ % (len(result),)
800
+ + result
801
+ )
802
+ self.pump()
803
+
804
+ # There should not be a second connection.
805
+ self.assertEqual(len(self.reactor.tcpClients), 1)
806
+
807
+ # The image should not be in the result.
808
+ self.assertEqual(channel.code, 200)
809
+ self.assertNotIn("og:image", channel.json_body)
810
+
811
+ def test_oembed_failure(self) -> None:
812
+ """If the autodiscovered oEmbed URL fails, ensure some data is returned."""
813
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
814
+
815
+ result = b"""
816
+ <title>oEmbed Autodiscovery Fail</title>
817
+ <link rel="alternate" type="application/json+oembed"
818
+ href="http://example.com/oembed?url=http%3A%2F%2Fmatrix.org&format=json"
819
+ title="matrixdotorg" />
820
+ """
821
+
822
+ channel = self.make_request(
823
+ "GET",
824
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
825
+ shorthand=False,
826
+ await_result=False,
827
+ )
828
+ self.pump()
829
+
830
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
831
+ server = AccumulatingProtocol()
832
+ server.makeConnection(FakeTransport(client, self.reactor))
833
+ client.makeConnection(FakeTransport(server, self.reactor))
834
+ client.dataReceived(
835
+ (
836
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
837
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
838
+ )
839
+ % (len(result),)
840
+ + result
841
+ )
842
+
843
+ self.pump()
844
+ self.assertEqual(channel.code, 200)
845
+
846
+ # The image should not be in the result.
847
+ self.assertEqual(channel.json_body["og:title"], "oEmbed Autodiscovery Fail")
848
+
849
+ def test_data_url(self) -> None:
850
+ """
851
+ Requesting to preview a data URL is not supported.
852
+ """
853
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
854
+
855
+ data = base64.b64encode(SMALL_PNG).decode()
856
+
857
+ query_params = urlencode(
858
+ {
859
+ "url": f'<html><head><img src="data:image/png;base64,{data}" /></head></html>'
860
+ }
861
+ )
862
+
863
+ channel = self.make_request(
864
+ "GET",
865
+ f"/_matrix/media/v3/preview_url?{query_params}",
866
+ shorthand=False,
867
+ )
868
+ self.pump()
869
+
870
+ self.assertEqual(channel.code, 500)
871
+
872
+ def test_inline_data_url(self) -> None:
873
+ """
874
+ An inline image (as a data URL) should be parsed properly.
875
+ """
876
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
877
+
878
+ data = base64.b64encode(SMALL_PNG)
879
+
880
+ end_content = (
881
+ b'<html><head><img src="data:image/png;base64,%s" /></head></html>'
882
+ ) % (data,)
883
+
884
+ channel = self.make_request(
885
+ "GET",
886
+ "/_matrix/media/v3/preview_url?url=http://matrix.org",
887
+ shorthand=False,
888
+ await_result=False,
889
+ )
890
+ self.pump()
891
+
892
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
893
+ server = AccumulatingProtocol()
894
+ server.makeConnection(FakeTransport(client, self.reactor))
895
+ client.makeConnection(FakeTransport(server, self.reactor))
896
+ client.dataReceived(
897
+ (
898
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
899
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
900
+ )
901
+ % (len(end_content),)
902
+ + end_content
903
+ )
904
+
905
+ self.pump()
906
+ self.assertEqual(channel.code, 200)
907
+ self._assert_small_png(channel.json_body)
908
+
909
+ def test_oembed_photo(self) -> None:
910
+ """Test an oEmbed endpoint which returns a 'photo' type which redirects the preview to a new URL."""
911
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
912
+ self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]
913
+
914
+ result = {
915
+ "version": "1.0",
916
+ "type": "photo",
917
+ "url": "http://cdn.twitter.com/matrixdotorg",
918
+ }
919
+ oembed_content = json.dumps(result).encode("utf-8")
920
+
921
+ channel = self.make_request(
922
+ "GET",
923
+ "/_matrix/media/v3/preview_url?url=http://twitter.com/matrixdotorg/status/12345",
924
+ shorthand=False,
925
+ await_result=False,
926
+ )
927
+ self.pump()
928
+
929
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
930
+ server = AccumulatingProtocol()
931
+ server.makeConnection(FakeTransport(client, self.reactor))
932
+ client.makeConnection(FakeTransport(server, self.reactor))
933
+ client.dataReceived(
934
+ (
935
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
936
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
937
+ )
938
+ % (len(oembed_content),)
939
+ + oembed_content
940
+ )
941
+
942
+ self.pump()
943
+
944
+ # Ensure a second request is made to the photo URL.
945
+ client = self.reactor.tcpClients[1][2].buildProtocol(None)
946
+ server = AccumulatingProtocol()
947
+ server.makeConnection(FakeTransport(client, self.reactor))
948
+ client.makeConnection(FakeTransport(server, self.reactor))
949
+ client.dataReceived(
950
+ (
951
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
952
+ b"Content-Type: image/png\r\n\r\n"
953
+ )
954
+ % (len(SMALL_PNG),)
955
+ + SMALL_PNG
956
+ )
957
+
958
+ self.pump()
959
+
960
+ # Ensure the URL is what was requested.
961
+ self.assertIn(b"/matrixdotorg", server.data)
962
+
963
+ self.assertEqual(channel.code, 200)
964
+ body = channel.json_body
965
+ self.assertEqual(body["og:url"], "http://twitter.com/matrixdotorg/status/12345")
966
+ self._assert_small_png(body)
967
+
968
+ def test_oembed_rich(self) -> None:
969
+ """Test an oEmbed endpoint which returns HTML content via the 'rich' type."""
970
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
971
+
972
+ result = {
973
+ "version": "1.0",
974
+ "type": "rich",
975
+ # Note that this provides the author, not the title.
976
+ "author_name": "Alice",
977
+ "html": "<div>Content Preview</div>",
978
+ }
979
+ end_content = json.dumps(result).encode("utf-8")
980
+
981
+ channel = self.make_request(
982
+ "GET",
983
+ "/_matrix/media/v3/preview_url?url=http://twitter.com/matrixdotorg/status/12345",
984
+ shorthand=False,
985
+ await_result=False,
986
+ )
987
+ self.pump()
988
+
989
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
990
+ server = AccumulatingProtocol()
991
+ server.makeConnection(FakeTransport(client, self.reactor))
992
+ client.makeConnection(FakeTransport(server, self.reactor))
993
+ client.dataReceived(
994
+ (
995
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
996
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
997
+ )
998
+ % (len(end_content),)
999
+ + end_content
1000
+ )
1001
+
1002
+ self.pump()
1003
+
1004
+ # Double check that the proper host is being connected to. (Note that
1005
+ # twitter.com can't be resolved so this is already implicitly checked.)
1006
+ self.assertIn(b"\r\nHost: publish.twitter.com\r\n", server.data)
1007
+
1008
+ self.assertEqual(channel.code, 200)
1009
+ body = channel.json_body
1010
+ self.assertEqual(
1011
+ body,
1012
+ {
1013
+ "og:url": "http://twitter.com/matrixdotorg/status/12345",
1014
+ "og:title": "Alice",
1015
+ "og:description": "Content Preview",
1016
+ },
1017
+ )
1018
+
1019
+ def test_oembed_format(self) -> None:
1020
+ """Test an oEmbed endpoint which requires the format in the URL."""
1021
+ self.lookups["www.hulu.com"] = [(IPv4Address, "10.1.2.3")]
1022
+
1023
+ result = {
1024
+ "version": "1.0",
1025
+ "type": "rich",
1026
+ "html": "<div>Content Preview</div>",
1027
+ }
1028
+ end_content = json.dumps(result).encode("utf-8")
1029
+
1030
+ channel = self.make_request(
1031
+ "GET",
1032
+ "/_matrix/media/v3/preview_url?url=http://www.hulu.com/watch/12345",
1033
+ shorthand=False,
1034
+ await_result=False,
1035
+ )
1036
+ self.pump()
1037
+
1038
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
1039
+ server = AccumulatingProtocol()
1040
+ server.makeConnection(FakeTransport(client, self.reactor))
1041
+ client.makeConnection(FakeTransport(server, self.reactor))
1042
+ client.dataReceived(
1043
+ (
1044
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
1045
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
1046
+ )
1047
+ % (len(end_content),)
1048
+ + end_content
1049
+ )
1050
+
1051
+ self.pump()
1052
+
1053
+ # The {format} should have been turned into json.
1054
+ self.assertIn(b"/api/oembed.json", server.data)
1055
+ # A URL parameter of format=json should be provided.
1056
+ self.assertIn(b"format=json", server.data)
1057
+
1058
+ self.assertEqual(channel.code, 200)
1059
+ body = channel.json_body
1060
+ self.assertEqual(
1061
+ body,
1062
+ {
1063
+ "og:url": "http://www.hulu.com/watch/12345",
1064
+ "og:description": "Content Preview",
1065
+ },
1066
+ )
1067
+
1068
+ @unittest.override_config(
1069
+ {"url_preview_url_blacklist": [{"netloc": "publish.twitter.com"}]}
1070
+ )
1071
+ def test_oembed_blocked(self) -> None:
1072
+ """The oEmbed URL should not be downloaded if the oEmbed URL is blocked."""
1073
+ self.lookups["twitter.com"] = [(IPv4Address, "10.1.2.3")]
1074
+
1075
+ channel = self.make_request(
1076
+ "GET",
1077
+ "/_matrix/media/v3/preview_url?url=http://twitter.com/matrixdotorg/status/12345",
1078
+ shorthand=False,
1079
+ await_result=False,
1080
+ )
1081
+ self.pump()
1082
+ self.assertEqual(channel.code, 403, channel.result)
1083
+
1084
+ def test_oembed_autodiscovery(self) -> None:
1085
+ """
1086
+ Autodiscovery works by finding the link in the HTML response and then requesting an oEmbed URL.
1087
+ 1. Request a preview of a URL which is not known to the oEmbed code.
1088
+ 2. It returns HTML including a link to an oEmbed preview.
1089
+ 3. The oEmbed preview is requested and returns a URL for an image.
1090
+ 4. The image is requested for thumbnailing.
1091
+ """
1092
+ # This is a little cheesy in that we use the www subdomain (which isn't the
1093
+ # list of oEmbed patterns) to get "raw" HTML response.
1094
+ self.lookups["www.twitter.com"] = [(IPv4Address, "10.1.2.3")]
1095
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
1096
+ self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]
1097
+
1098
+ result = b"""
1099
+ <link rel="alternate" type="application/json+oembed"
1100
+ href="http://publish.twitter.com/oembed?url=http%3A%2F%2Fcdn.twitter.com%2Fmatrixdotorg%2Fstatus%2F12345&format=json"
1101
+ title="matrixdotorg" />
1102
+ """
1103
+
1104
+ channel = self.make_request(
1105
+ "GET",
1106
+ "/_matrix/media/v3/preview_url?url=http://www.twitter.com/matrixdotorg/status/12345",
1107
+ shorthand=False,
1108
+ await_result=False,
1109
+ )
1110
+ self.pump()
1111
+
1112
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
1113
+ server = AccumulatingProtocol()
1114
+ server.makeConnection(FakeTransport(client, self.reactor))
1115
+ client.makeConnection(FakeTransport(server, self.reactor))
1116
+ client.dataReceived(
1117
+ (
1118
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
1119
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
1120
+ )
1121
+ % (len(result),)
1122
+ + result
1123
+ )
1124
+ self.pump()
1125
+
1126
+ # The oEmbed response.
1127
+ result2 = {
1128
+ "version": "1.0",
1129
+ "type": "photo",
1130
+ "url": "http://cdn.twitter.com/matrixdotorg",
1131
+ }
1132
+ oembed_content = json.dumps(result2).encode("utf-8")
1133
+
1134
+ # Ensure a second request is made to the oEmbed URL.
1135
+ client = self.reactor.tcpClients[1][2].buildProtocol(None)
1136
+ server = AccumulatingProtocol()
1137
+ server.makeConnection(FakeTransport(client, self.reactor))
1138
+ client.makeConnection(FakeTransport(server, self.reactor))
1139
+ client.dataReceived(
1140
+ (
1141
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
1142
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
1143
+ )
1144
+ % (len(oembed_content),)
1145
+ + oembed_content
1146
+ )
1147
+ self.pump()
1148
+
1149
+ # Ensure the URL is what was requested.
1150
+ self.assertIn(b"/oembed?", server.data)
1151
+
1152
+ # Ensure a third request is made to the photo URL.
1153
+ client = self.reactor.tcpClients[2][2].buildProtocol(None)
1154
+ server = AccumulatingProtocol()
1155
+ server.makeConnection(FakeTransport(client, self.reactor))
1156
+ client.makeConnection(FakeTransport(server, self.reactor))
1157
+ client.dataReceived(
1158
+ (
1159
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
1160
+ b"Content-Type: image/png\r\n\r\n"
1161
+ )
1162
+ % (len(SMALL_PNG),)
1163
+ + SMALL_PNG
1164
+ )
1165
+ self.pump()
1166
+
1167
+ # Ensure the URL is what was requested.
1168
+ self.assertIn(b"/matrixdotorg", server.data)
1169
+
1170
+ self.assertEqual(channel.code, 200)
1171
+ body = channel.json_body
1172
+ self.assertEqual(
1173
+ body["og:url"], "http://www.twitter.com/matrixdotorg/status/12345"
1174
+ )
1175
+ self._assert_small_png(body)
1176
+
1177
+ @unittest.override_config(
1178
+ {"url_preview_url_blacklist": [{"netloc": "publish.twitter.com"}]}
1179
+ )
1180
+ def test_oembed_autodiscovery_blocked(self) -> None:
1181
+ """
1182
+ If the discovered oEmbed URL is blocked, it should be discarded.
1183
+ """
1184
+ # This is a little cheesy in that we use the www subdomain (which isn't the
1185
+ # list of oEmbed patterns) to get "raw" HTML response.
1186
+ self.lookups["www.twitter.com"] = [(IPv4Address, "10.1.2.3")]
1187
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.4")]
1188
+
1189
+ result = b"""
1190
+ <title>Test</title>
1191
+ <link rel="alternate" type="application/json+oembed"
1192
+ href="http://publish.twitter.com/oembed?url=http%3A%2F%2Fcdn.twitter.com%2Fmatrixdotorg%2Fstatus%2F12345&format=json"
1193
+ title="matrixdotorg" />
1194
+ """
1195
+
1196
+ channel = self.make_request(
1197
+ "GET",
1198
+ "/_matrix/media/v3/preview_url?url=http://www.twitter.com/matrixdotorg/status/12345",
1199
+ shorthand=False,
1200
+ await_result=False,
1201
+ )
1202
+ self.pump()
1203
+
1204
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
1205
+ server = AccumulatingProtocol()
1206
+ server.makeConnection(FakeTransport(client, self.reactor))
1207
+ client.makeConnection(FakeTransport(server, self.reactor))
1208
+ client.dataReceived(
1209
+ (
1210
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
1211
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
1212
+ )
1213
+ % (len(result),)
1214
+ + result
1215
+ )
1216
+
1217
+ self.pump()
1218
+
1219
+ # Ensure there's no additional connections.
1220
+ self.assertEqual(len(self.reactor.tcpClients), 1)
1221
+
1222
+ # Ensure the URL is what was requested.
1223
+ self.assertIn(b"\r\nHost: www.twitter.com\r\n", server.data)
1224
+
1225
+ self.assertEqual(channel.code, 200)
1226
+ body = channel.json_body
1227
+ self.assertEqual(body["og:title"], "Test")
1228
+ self.assertNotIn("og:image", body)
1229
+
1230
+ def _download_image(self) -> tuple[str, str]:
1231
+ """Downloads an image into the URL cache.
1232
+ Returns:
1233
+ A (host, media_id) tuple representing the MXC URI of the image.
1234
+ """
1235
+ self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]
1236
+
1237
+ channel = self.make_request(
1238
+ "GET",
1239
+ "/_matrix/media/v3/preview_url?url=http://cdn.twitter.com/matrixdotorg",
1240
+ shorthand=False,
1241
+ await_result=False,
1242
+ )
1243
+ self.pump()
1244
+
1245
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
1246
+ server = AccumulatingProtocol()
1247
+ server.makeConnection(FakeTransport(client, self.reactor))
1248
+ client.makeConnection(FakeTransport(server, self.reactor))
1249
+ client.dataReceived(
1250
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: image/png\r\n\r\n"
1251
+ % (len(SMALL_PNG),)
1252
+ + SMALL_PNG
1253
+ )
1254
+
1255
+ self.pump()
1256
+ self.assertEqual(channel.code, 200)
1257
+ body = channel.json_body
1258
+ mxc_uri = body["og:image"]
1259
+ host, _port, media_id = parse_and_validate_mxc_uri(mxc_uri)
1260
+ self.assertIsNone(_port)
1261
+ return host, media_id
1262
+
1263
+ @override_config({"enable_authenticated_media": False})
1264
+ def test_storage_providers_exclude_files(self) -> None:
1265
+ """Test that files are not stored in or fetched from storage providers."""
1266
+ host, media_id = self._download_image()
1267
+
1268
+ rel_file_path = self.media_repo.filepaths.url_cache_filepath_rel(media_id)
1269
+ media_store_path = os.path.join(self.media_store_path, rel_file_path)
1270
+ storage_provider_path = os.path.join(self.storage_path, rel_file_path)
1271
+
1272
+ # Check storage
1273
+ self.assertTrue(os.path.isfile(media_store_path))
1274
+ self.assertFalse(
1275
+ os.path.isfile(storage_provider_path),
1276
+ "URL cache file was unexpectedly stored in a storage provider",
1277
+ )
1278
+
1279
+ # Check fetching
1280
+ channel = self.make_request(
1281
+ "GET",
1282
+ f"/_matrix/media/v3/download/{host}/{media_id}",
1283
+ shorthand=False,
1284
+ await_result=False,
1285
+ )
1286
+ self.pump()
1287
+ self.assertEqual(channel.code, 200)
1288
+
1289
+ # Move cached file into the storage provider
1290
+ os.makedirs(os.path.dirname(storage_provider_path), exist_ok=True)
1291
+ os.rename(media_store_path, storage_provider_path)
1292
+
1293
+ channel = self.make_request(
1294
+ "GET",
1295
+ f"/_matrix/media/v3/download/{host}/{media_id}",
1296
+ shorthand=False,
1297
+ await_result=False,
1298
+ )
1299
+ self.pump()
1300
+ self.assertEqual(
1301
+ channel.code,
1302
+ 404,
1303
+ "URL cache file was unexpectedly retrieved from a storage provider",
1304
+ )
1305
+
1306
+ @override_config({"enable_authenticated_media": False})
1307
+ def test_storage_providers_exclude_thumbnails(self) -> None:
1308
+ """Test that thumbnails are not stored in or fetched from storage providers."""
1309
+ host, media_id = self._download_image()
1310
+
1311
+ rel_thumbnail_path = (
1312
+ self.media_repo.filepaths.url_cache_thumbnail_directory_rel(media_id)
1313
+ )
1314
+ media_store_thumbnail_path = os.path.join(
1315
+ self.media_store_path, rel_thumbnail_path
1316
+ )
1317
+ storage_provider_thumbnail_path = os.path.join(
1318
+ self.storage_path, rel_thumbnail_path
1319
+ )
1320
+
1321
+ # Check storage
1322
+ self.assertTrue(os.path.isdir(media_store_thumbnail_path))
1323
+ self.assertFalse(
1324
+ os.path.isdir(storage_provider_thumbnail_path),
1325
+ "URL cache thumbnails were unexpectedly stored in a storage provider",
1326
+ )
1327
+
1328
+ # Check fetching
1329
+ channel = self.make_request(
1330
+ "GET",
1331
+ f"/_matrix/media/v3/thumbnail/{host}/{media_id}?width=32&height=32&method=scale",
1332
+ shorthand=False,
1333
+ await_result=False,
1334
+ )
1335
+ self.pump()
1336
+ self.assertEqual(channel.code, 200)
1337
+
1338
+ # Remove the original, otherwise thumbnails will regenerate
1339
+ rel_file_path = self.media_repo.filepaths.url_cache_filepath_rel(media_id)
1340
+ media_store_path = os.path.join(self.media_store_path, rel_file_path)
1341
+ os.remove(media_store_path)
1342
+
1343
+ # Move cached thumbnails into the storage provider
1344
+ os.makedirs(os.path.dirname(storage_provider_thumbnail_path), exist_ok=True)
1345
+ os.rename(media_store_thumbnail_path, storage_provider_thumbnail_path)
1346
+
1347
+ channel = self.make_request(
1348
+ "GET",
1349
+ f"/_matrix/media/v3/thumbnail/{host}/{media_id}?width=32&height=32&method=scale",
1350
+ shorthand=False,
1351
+ await_result=False,
1352
+ )
1353
+ self.pump()
1354
+ self.assertEqual(
1355
+ channel.code,
1356
+ 404,
1357
+ "URL cache thumbnail was unexpectedly retrieved from a storage provider",
1358
+ )
1359
+
1360
+ def test_cache_expiry(self) -> None:
1361
+ """Test that URL cache files and thumbnails are cleaned up properly on expiry."""
1362
+ _host, media_id = self._download_image()
1363
+
1364
+ file_path = self.media_repo.filepaths.url_cache_filepath(media_id)
1365
+ file_dirs = self.media_repo.filepaths.url_cache_filepath_dirs_to_delete(
1366
+ media_id
1367
+ )
1368
+ thumbnail_dir = self.media_repo.filepaths.url_cache_thumbnail_directory(
1369
+ media_id
1370
+ )
1371
+ thumbnail_dirs = self.media_repo.filepaths.url_cache_thumbnail_dirs_to_delete(
1372
+ media_id
1373
+ )
1374
+
1375
+ self.assertTrue(os.path.isfile(file_path))
1376
+ self.assertTrue(os.path.isdir(thumbnail_dir))
1377
+
1378
+ self.reactor.advance(IMAGE_CACHE_EXPIRY_MS * 1000 + 1)
1379
+ self.get_success(self.url_previewer._expire_url_cache_data())
1380
+
1381
+ for path in [file_path] + file_dirs + [thumbnail_dir] + thumbnail_dirs:
1382
+ self.assertFalse(
1383
+ os.path.exists(path),
1384
+ f"{os.path.relpath(path, self.media_store_path)} was not deleted",
1385
+ )
1386
+
1387
+ @unittest.override_config({"url_preview_url_blacklist": [{"port": "*"}]})
1388
+ def test_blocked_port(self) -> None:
1389
+ """Tests that blocking URLs with a port makes previewing such URLs
1390
+ fail with a 403 error and doesn't impact other previews.
1391
+ """
1392
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
1393
+
1394
+ bad_url = quote("http://matrix.org:8888/foo")
1395
+ good_url = quote("http://matrix.org/foo")
1396
+
1397
+ channel = self.make_request(
1398
+ "GET",
1399
+ "/_matrix/media/v3/preview_url?url=" + bad_url,
1400
+ shorthand=False,
1401
+ await_result=False,
1402
+ )
1403
+ self.pump()
1404
+ self.assertEqual(channel.code, 403, channel.result)
1405
+
1406
+ channel = self.make_request(
1407
+ "GET",
1408
+ "/_matrix/media/v3/preview_url?url=" + good_url,
1409
+ shorthand=False,
1410
+ await_result=False,
1411
+ )
1412
+ self.pump()
1413
+
1414
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
1415
+ server = AccumulatingProtocol()
1416
+ server.makeConnection(FakeTransport(client, self.reactor))
1417
+ client.makeConnection(FakeTransport(server, self.reactor))
1418
+ client.dataReceived(
1419
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
1420
+ % (len(self.end_content),)
1421
+ + self.end_content
1422
+ )
1423
+
1424
+ self.pump()
1425
+ self.assertEqual(channel.code, 200)
1426
+
1427
+ @unittest.override_config(
1428
+ {"url_preview_url_blacklist": [{"netloc": "example.com"}]}
1429
+ )
1430
+ def test_blocked_url(self) -> None:
1431
+ """Tests that blocking URLs with a host makes previewing such URLs
1432
+ fail with a 403 error.
1433
+ """
1434
+ self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")]
1435
+
1436
+ bad_url = quote("http://example.com/foo")
1437
+
1438
+ channel = self.make_request(
1439
+ "GET",
1440
+ "/_matrix/media/v3/preview_url?url=" + bad_url,
1441
+ shorthand=False,
1442
+ await_result=False,
1443
+ )
1444
+ self.pump()
1445
+ self.assertEqual(channel.code, 403, channel.result)