pal-code 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (282) hide show
  1. pal/__init__.py +29 -0
  2. pal/__main__.py +3 -0
  3. pal/access_manager/__init__.py +16 -0
  4. pal/access_manager/components/__init__.py +14 -0
  5. pal/access_manager/components/access_control.py +41 -0
  6. pal/access_manager/components/base.py +39 -0
  7. pal/access_manager/components/intervention.py +31 -0
  8. pal/access_manager/components/privilege_registry.py +30 -0
  9. pal/access_manager/pipeline.py +68 -0
  10. pal/access_manager/utils/__init__.py +2 -0
  11. pal/agent_constraints/__init__.py +3 -0
  12. pal/agent_constraints/components/AGENT_OUTPUT.md +13 -0
  13. pal/agent_constraints/components/__init__.py +3 -0
  14. pal/agent_constraints/components/constraints.py +22 -0
  15. pal/agent_constraints/pipeline.py +22 -0
  16. pal/agent_constraints/utils/__init__.py +2 -0
  17. pal/agents/__init__.py +12 -0
  18. pal/agents/coding.py +14 -0
  19. pal/agents/docs.py +15 -0
  20. pal/chat/__init__.py +4 -0
  21. pal/chat/demo.py +231 -0
  22. pal/chat/providers.py +23 -0
  23. pal/chat/service.py +591 -0
  24. pal/chat/services/__init__.py +26 -0
  25. pal/chat/services/base.py +142 -0
  26. pal/chat/services/data.py +142 -0
  27. pal/chat/services/media.py +123 -0
  28. pal/chat/services/web.py +60 -0
  29. pal/chat/sessions.py +108 -0
  30. pal/chat/tools.py +446 -0
  31. pal/chat/uploads.py +81 -0
  32. pal/cli.py +502 -0
  33. pal/compression/__init__.py +12 -0
  34. pal/compression/components/__init__.py +64 -0
  35. pal/compression/components/adverbs.py +20 -0
  36. pal/compression/components/hedges.py +54 -0
  37. pal/compression/components/interjections.py +20 -0
  38. pal/compression/components/minifier.py +56 -0
  39. pal/compression/components/output_compressor.py +13 -0
  40. pal/compression/components/pleonasms.py +54 -0
  41. pal/compression/components/punctuations.py +36 -0
  42. pal/compression/components/stopwords.py +62 -0
  43. pal/compression/history.py +346 -0
  44. pal/compression/pipeline.py +58 -0
  45. pal/compression/utils/__init__.py +30 -0
  46. pal/compression/utils/negations.py +18 -0
  47. pal/compression/utils/nltk_setup.py +20 -0
  48. pal/compression/utils/pronouns.py +27 -0
  49. pal/compression/utils/proselint_setup.py +9 -0
  50. pal/compression/utils/spacy_setup.py +22 -0
  51. pal/config/default.yaml +163 -0
  52. pal/config.py +504 -0
  53. pal/context_manager/__init__.py +7 -0
  54. pal/context_manager/chat_context.py +117 -0
  55. pal/context_manager/components/__init__.py +5 -0
  56. pal/context_manager/components/assembler.py +75 -0
  57. pal/context_manager/components/context_cache.py +45 -0
  58. pal/context_manager/components/kv_bridge.py +16 -0
  59. pal/context_manager/components/pager.py +48 -0
  60. pal/context_manager/components/simple_context_manager.py +56 -0
  61. pal/context_manager/components/turns.py +88 -0
  62. pal/context_manager/pipeline.py +27 -0
  63. pal/context_manager/utils/__init__.py +18 -0
  64. pal/context_manager/utils/hashing.py +5 -0
  65. pal/context_manager/utils/relevance.py +59 -0
  66. pal/context_manager/utils/storage.py +37 -0
  67. pal/documents/__init__.py +4 -0
  68. pal/documents/components/__init__.py +0 -0
  69. pal/documents/components/chunker.py +79 -0
  70. pal/documents/components/index.py +155 -0
  71. pal/documents/utils/__init__.py +1 -0
  72. pal/errors.py +40 -0
  73. pal/input_converters/__init__.py +8 -0
  74. pal/input_converters/components/__init__.py +0 -0
  75. pal/input_converters/components/base.py +38 -0
  76. pal/input_converters/components/pandoc_converter.py +87 -0
  77. pal/input_converters/components/pdf_converter.py +46 -0
  78. pal/input_converters/components/pptx_converter.py +52 -0
  79. pal/input_converters/components/xlsx_converter.py +38 -0
  80. pal/input_converters/pipeline.py +72 -0
  81. pal/input_converters/utils/__init__.py +1 -0
  82. pal/input_converters/workspace.py +56 -0
  83. pal/input_savers/INPUT_RULES.md +2 -0
  84. pal/input_savers/__init__.py +5 -0
  85. pal/input_savers/components/__init__.py +0 -0
  86. pal/input_savers/components/base.py +40 -0
  87. pal/input_savers/components/methods.py +41 -0
  88. pal/input_savers/components/verifier.py +39 -0
  89. pal/input_savers/pipeline.py +154 -0
  90. pal/input_savers/utils/__init__.py +0 -0
  91. pal/input_savers/utils/guards.py +94 -0
  92. pal/knowledge_graph/__init__.py +10 -0
  93. pal/knowledge_graph/components/__init__.py +0 -0
  94. pal/knowledge_graph/components/base.py +50 -0
  95. pal/knowledge_graph/components/builder.py +54 -0
  96. pal/knowledge_graph/components/retriever.py +100 -0
  97. pal/knowledge_graph/components/workspace.py +116 -0
  98. pal/knowledge_graph/pipeline.py +215 -0
  99. pal/knowledge_graph/utils/__init__.py +1 -0
  100. pal/latency_tracker/__init__.py +3 -0
  101. pal/latency_tracker/pipeline.py +86 -0
  102. pal/llm_scheduler/__init__.py +11 -0
  103. pal/llm_scheduler/api.py +137 -0
  104. pal/llm_scheduler/code.py +28 -0
  105. pal/llm_scheduler/core.py +92 -0
  106. pal/llm_scheduler/local.py +215 -0
  107. pal/llm_scheduler/pipeline.py +42 -0
  108. pal/llm_scheduler/types.py +92 -0
  109. pal/llm_wrapper/__init__.py +34 -0
  110. pal/llm_wrapper/adapter.py +316 -0
  111. pal/llm_wrapper/components/__init__.py +23 -0
  112. pal/llm_wrapper/components/adapter_types.py +90 -0
  113. pal/llm_wrapper/components/base.py +195 -0
  114. pal/llm_wrapper/components/features.py +104 -0
  115. pal/llm_wrapper/components/litellm_wrapper.py +215 -0
  116. pal/llm_wrapper/components/openrouter_wrapper.py +114 -0
  117. pal/llm_wrapper/modes.py +73 -0
  118. pal/llm_wrapper/pipeline.py +61 -0
  119. pal/llm_wrapper/utils/__init__.py +4 -0
  120. pal/llm_wrapper/utils/registry.py +49 -0
  121. pal/llm_wrapper/utils/token_counting.py +10 -0
  122. pal/local_llm/__init__.py +5 -0
  123. pal/local_llm/components/__init__.py +0 -0
  124. pal/local_llm/components/huggingface_local.py +217 -0
  125. pal/local_llm/components/local_llm.py +60 -0
  126. pal/logging_setup.py +18 -0
  127. pal/memory/__init__.py +7 -0
  128. pal/memory/components/__init__.py +6 -0
  129. pal/memory/components/lru_k.py +56 -0
  130. pal/memory/components/manager.py +98 -0
  131. pal/memory/components/note.py +54 -0
  132. pal/memory/components/storage_manager.py +60 -0
  133. pal/memory/pipeline.py +122 -0
  134. pal/memory/utils/__init__.py +3 -0
  135. pal/memory/utils/keyword_overlap.py +3 -0
  136. pal/memory_tracker/__init__.py +3 -0
  137. pal/memory_tracker/components/__init__.py +0 -0
  138. pal/memory_tracker/components/process.py +56 -0
  139. pal/memory_tracker/pipeline.py +44 -0
  140. pal/metrics/__init__.py +4 -0
  141. pal/metrics/components/__init__.py +4 -0
  142. pal/metrics/components/collector.py +140 -0
  143. pal/metrics/components/exporter.py +28 -0
  144. pal/metrics/pipeline.py +49 -0
  145. pal/metrics/utils/__init__.py +2 -0
  146. pal/notes_manager/__init__.py +5 -0
  147. pal/notes_manager/components/__init__.py +0 -0
  148. pal/notes_manager/components/privacy.py +14 -0
  149. pal/notes_manager/components/recall.py +31 -0
  150. pal/notes_manager/components/tidy.py +28 -0
  151. pal/notes_manager/pipeline.py +138 -0
  152. pal/orchestrator/__init__.py +26 -0
  153. pal/orchestrator/components/__init__.py +26 -0
  154. pal/orchestrator/components/agent.py +105 -0
  155. pal/orchestrator/components/cgroup_manager.py +209 -0
  156. pal/orchestrator/components/context_interrupt.py +220 -0
  157. pal/orchestrator/components/limits.py +9 -0
  158. pal/orchestrator/components/scheduler.py +69 -0
  159. pal/orchestrator/components/syscall_scheduler.py +200 -0
  160. pal/orchestrator/pipeline.py +228 -0
  161. pal/orchestrator/utils/__init__.py +4 -0
  162. pal/orchestrator/utils/device.py +164 -0
  163. pal/orchestrator/utils/protocol.py +32 -0
  164. pal/os_kernel/__init__.py +5 -0
  165. pal/os_kernel/filesystem.py +80 -0
  166. pal/os_kernel/network.py +146 -0
  167. pal/os_kernel/pipeline.py +175 -0
  168. pal/os_kernel/sysinfo.py +36 -0
  169. pal/paths.py +59 -0
  170. pal/pipeline.py +177 -0
  171. pal/py.typed +0 -0
  172. pal/redact.py +19 -0
  173. pal/sdk/__init__.py +33 -0
  174. pal/sdk/adapters.py +43 -0
  175. pal/sdk/agent.py +99 -0
  176. pal/sdk/components/__init__.py +14 -0
  177. pal/sdk/components/autogen_adapter.py +65 -0
  178. pal/sdk/components/autogen_client.py +98 -0
  179. pal/sdk/components/framework_adapter_base.py +69 -0
  180. pal/sdk/components/metagpt_adapter.py +46 -0
  181. pal/sdk/components/open_interpreter_adapter.py +47 -0
  182. pal/sdk/components/tool_catalog.py +44 -0
  183. pal/sdk/os_api.py +84 -0
  184. pal/sdk/patterns.py +129 -0
  185. pal/sdk/pipeline.py +240 -0
  186. pal/sdk/queries.py +68 -0
  187. pal/sdk/savers.py +88 -0
  188. pal/sdk/utils/__init__.py +2 -0
  189. pal/storage_manager/__init__.py +17 -0
  190. pal/storage_manager/components/__init__.py +19 -0
  191. pal/storage_manager/components/base.py +50 -0
  192. pal/storage_manager/components/local_file_backend.py +84 -0
  193. pal/storage_manager/components/storage_queue.py +91 -0
  194. pal/storage_manager/components/vector_store_backend.py +122 -0
  195. pal/storage_manager/pipeline.py +153 -0
  196. pal/storage_manager/utils/__init__.py +4 -0
  197. pal/storage_manager/utils/memory_adapter.py +60 -0
  198. pal/storage_manager/utils/sdk.py +93 -0
  199. pal/studio/__init__.py +0 -0
  200. pal/studio/builder.py +166 -0
  201. pal/studio/catalog.py +90 -0
  202. pal/studio/checks.py +129 -0
  203. pal/studio/clients.py +6 -0
  204. pal/studio/editor.py +137 -0
  205. pal/studio/files.py +6 -0
  206. pal/studio/models.py +6 -0
  207. pal/studio/plan.py +149 -0
  208. pal/studio/planner.py +205 -0
  209. pal/studio/policy.py +69 -0
  210. pal/studio/routes.py +254 -0
  211. pal/studio/runs.py +210 -0
  212. pal/task_router/__init__.py +11 -0
  213. pal/task_router/components/__init__.py +0 -0
  214. pal/task_router/components/answerer.py +37 -0
  215. pal/task_router/components/base.py +34 -0
  216. pal/task_router/components/classifier.py +40 -0
  217. pal/task_router/components/gates.py +23 -0
  218. pal/task_router/pipeline.py +61 -0
  219. pal/task_router/utils/__init__.py +1 -0
  220. pal/terminal/__init__.py +1 -0
  221. pal/terminal/backend.py +141 -0
  222. pal/terminal/bench.py +173 -0
  223. pal/terminal/cli.py +826 -0
  224. pal/terminal/clients.py +170 -0
  225. pal/terminal/commands.py +401 -0
  226. pal/terminal/decide.py +242 -0
  227. pal/terminal/files.py +55 -0
  228. pal/terminal/flow_cli.py +439 -0
  229. pal/terminal/hardware.py +199 -0
  230. pal/terminal/history.py +88 -0
  231. pal/terminal/keys.py +83 -0
  232. pal/terminal/local_models.py +261 -0
  233. pal/terminal/logo.py +117 -0
  234. pal/terminal/models.py +99 -0
  235. pal/terminal/policy.py +54 -0
  236. pal/terminal/project.py +413 -0
  237. pal/terminal/review.py +65 -0
  238. pal/terminal/security.py +239 -0
  239. pal/terminal/session.py +503 -0
  240. pal/terminal/setup.py +354 -0
  241. pal/terminal/syntax.py +98 -0
  242. pal/terminal/theme.py +37 -0
  243. pal/terminal/ui.py +647 -0
  244. pal/terminal/voice.py +137 -0
  245. pal/terminal/workflow.py +624 -0
  246. pal/token_tracker/__init__.py +3 -0
  247. pal/token_tracker/pipeline.py +54 -0
  248. pal/tokens.py +21 -0
  249. pal/tool_catalog.py +107 -0
  250. pal/tool_manager/__init__.py +17 -0
  251. pal/tool_manager/builtin.py +82 -0
  252. pal/tool_manager/components/__init__.py +19 -0
  253. pal/tool_manager/components/base.py +53 -0
  254. pal/tool_manager/components/conflict_resolver.py +27 -0
  255. pal/tool_manager/components/param_validator.py +41 -0
  256. pal/tool_manager/components/tool_call_queue.py +111 -0
  257. pal/tool_manager/components/tool_registry.py +60 -0
  258. pal/tool_manager/pipeline.py +80 -0
  259. pal/tool_manager/utils/__init__.py +3 -0
  260. pal/web/__init__.py +3 -0
  261. pal/web/app.py +276 -0
  262. pal/web/static/editor.js +219 -0
  263. pal/web/static/fonts/LICENSE-Inter.txt +3 -0
  264. pal/web/static/fonts/inter-400.woff2 +0 -0
  265. pal/web/static/fonts/inter-500.woff2 +0 -0
  266. pal/web/static/fonts/inter-600.woff2 +0 -0
  267. pal/web/static/fonts/inter-700.woff2 +0 -0
  268. pal/web/static/inter.css +5 -0
  269. pal/web/static/pal-logo-256.png +0 -0
  270. pal/web/static/pal-logo-32.png +0 -0
  271. pal/web/static/pal-logo-64.png +0 -0
  272. pal/web/static/studio.css +306 -0
  273. pal/web/static/studio.js +533 -0
  274. pal/web/static/voice.js +76 -0
  275. pal/web/templates/index.html +368 -0
  276. pal/web/templates/studio.html +120 -0
  277. pal_code-0.1.0.dist-info/METADATA +379 -0
  278. pal_code-0.1.0.dist-info/RECORD +282 -0
  279. pal_code-0.1.0.dist-info/WHEEL +5 -0
  280. pal_code-0.1.0.dist-info/entry_points.txt +2 -0
  281. pal_code-0.1.0.dist-info/licenses/LICENSE +21 -0
  282. pal_code-0.1.0.dist-info/top_level.txt +1 -0
pal/terminal/policy.py ADDED
@@ -0,0 +1,54 @@
1
+ """Which paths and files Pal may write. Enforced in code, with every refusal explained.
2
+
3
+ A model's reply is untrusted text. Before any file is written its path goes through `safe_path`: it must stay inside the folder
4
+ (no `..`, absolute paths, drive letters, hidden files, reserved Windows names), the kind of file must be allowed (programs and scripts are not: you add
5
+ those yourself). Paths are compared without regard to case, because Windows and macOS treat `Index.html` and
6
+ `index.html` as one file.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from pathlib import PurePosixPath
12
+
13
+ ALLOWED_EXTENSIONS = frozenset({".html", ".htm", ".css", ".js", ".mjs", ".json", ".md", ".txt", ".py", ".csv", ".svg", ".yml", ".yaml", ".toml", ".ini", ".cfg",
14
+ ".xml", ".ts", ".tsx", ".jsx", ".sql"})
15
+ ALLOWED_NAMES = frozenset({".gitignore"})
16
+ MAX_FILE_BYTES, MAX_FILES, MAX_TOTAL_BYTES = 200_000, 60, 2_000_000
17
+ _RESERVED = frozenset({"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))})
18
+ _PART = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_. \-]*$")
19
+ _DRIVE = re.compile(r"^[A-Za-z]:")
20
+
21
+
22
+ class PolicyError(ValueError):
23
+ """A path or file the policy refuses; the message says why, in plain words."""
24
+
25
+
26
+ def safe_path(raw: str, extensions: frozenset = ALLOWED_EXTENSIONS, names: frozenset = ALLOWED_NAMES) -> str:
27
+ """The path as a clean relative POSIX path inside the project folder, or PolicyError. `extensions` and `names` say which kinds of file are allowed."""
28
+ if not isinstance(raw, str) or not raw.strip():
29
+ raise PolicyError("the file has no name")
30
+ text = raw.strip().replace("\\", "/")
31
+ if "\x00" in text or any(ord(c) < 32 for c in text):
32
+ raise PolicyError("the file name has control characters")
33
+ if text.startswith("/") or _DRIVE.match(text):
34
+ raise PolicyError("absolute paths are not allowed: files go inside the project folder")
35
+ parts = [p for p in text.split("/") if p not in ("", ".")]
36
+ if not parts:
37
+ raise PolicyError("the file has no name")
38
+ if any(p == ".." for p in parts):
39
+ raise PolicyError("'..' is not allowed: files must stay inside the project folder")
40
+ if len(text) > 120 or any(len(p) > 60 for p in parts):
41
+ raise PolicyError("the file name is too long")
42
+ for p in parts:
43
+ if p.rstrip(". ") != p:
44
+ raise PolicyError("a name may not end with a dot or a space")
45
+ if p.split(".")[0].lower() in _RESERVED:
46
+ raise PolicyError(f"'{p}' is a reserved name on Windows")
47
+ if p.startswith(".") and p not in names:
48
+ raise PolicyError("hidden files are not allowed")
49
+ if p not in names and not _PART.match(p):
50
+ raise PolicyError(f"'{p}' has characters that are not allowed")
51
+ name = parts[-1]
52
+ if name not in names and PurePosixPath(name).suffix.lower() not in extensions:
53
+ raise PolicyError(f"'{name}': this kind of file is not allowed (programs and scripts are added by you, not by the bot)")
54
+ return "/".join(parts)
@@ -0,0 +1,413 @@
1
+ """Setting up the folder Pal works in, when `pal cli` or `pal web` starts.
2
+
3
+ Four things are prepared for the current folder and kept in ~/.pal/projects/<folder id>/, and only what changed is redone next time:
4
+
5
+ documents PDF, Word, PowerPoint, Excel, EPUB, RTF and LaTeX files are converted to markdown files (markdown/<path>.md). Markdown and
6
+ text files are used as they are. They are indexed, so each request carries only the passages that match it.
7
+ project map a short outline of the code: every file with its classes and functions. It is indexed too, so a question about
8
+ "the login handler" brings the lines of the map that name it, not the whole project.
9
+ code graph built with graphify from the source (AST only: no model, nothing leaves the computer). Each request gets the
10
+ graph nodes and code excerpts that match it. Built in the background, so the session opens at once.
11
+ brief languages, size, the test command: a few lines the model gets with the first request.
12
+
13
+ Nothing is written in the folder itself.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import ast
18
+ import fnmatch
19
+ import json
20
+ import os
21
+ import re
22
+ import shutil
23
+ import threading
24
+ import time
25
+ from collections import Counter
26
+ from dataclasses import dataclass, field
27
+ from pathlib import Path, PurePosixPath
28
+
29
+ from pal.knowledge_graph import KnowledgeGraph
30
+ from pal.knowledge_graph.components import builder
31
+ from pal.knowledge_graph.components.workspace import CODE_EXTENSIONS, SKIP_DIRS, Workspace
32
+ from pal.paths import pal_home
33
+ from pal.terminal.security import is_secret_file
34
+ from pal.terminal.workflow import project_id
35
+ from pal.tokens import estimate_tokens
36
+
37
+ DOC_EXTENSIONS = {".pdf", ".docx", ".pptx", ".xlsx", ".odt", ".rtf", ".epub", ".tex"} # converted to markdown
38
+ TEXT_DOC_EXTENSIONS = {".md", ".markdown", ".txt", ".rst"} # already text: used as they are
39
+ IGNORE_FILE = ".palignore"
40
+ NOTES_FILE = "PAL.md" # the project's instructions for Pal, sent with the first request of every session
41
+ MAX_NOTES_CHARS = 8000
42
+ MAX_CODE_FILES = 5000
43
+ MAX_DOCS = 300
44
+ MAX_DOC_BYTES = 25_000_000
45
+ MAX_CODE_BYTES = 1_000_000
46
+ MAP_ENTRIES_PER_FILE = 40
47
+ GRAPH_WORKSPACE = "project"
48
+ LANGUAGES = {".py": "Python", ".js": "JavaScript", ".jsx": "JavaScript", ".mjs": "JavaScript", ".ts": "TypeScript", ".tsx": "TypeScript",
49
+ ".go": "Go", ".rs": "Rust", ".java": "Java", ".kt": "Kotlin", ".kts": "Kotlin", ".swift": "Swift", ".rb": "Ruby", ".php": "PHP",
50
+ ".cs": "C#", ".c": "C", ".h": "C", ".cpp": "C++", ".cc": "C++", ".hpp": "C++", ".scala": "Scala", ".lua": "Lua", ".dart": "Dart",
51
+ ".ex": "Elixir", ".exs": "Elixir", ".jl": "Julia", ".zig": "Zig", ".sh": "Shell"}
52
+
53
+ # a definition line in languages other than Python: good enough for an outline
54
+ _DEF = re.compile(r"^\s*(?:export\s+)?(?:default\s+)?(?:public\s+|private\s+|protected\s+|internal\s+|static\s+|async\s+|abstract\s+|final\s+|open\s+|override\s+|pub(?:\(crate\))?\s+)*"
55
+ r"(?P<kind>class|interface|struct|enum|trait|type|function|func|fn|fun|def|object|module|impl)\s+(?P<name>[A-Za-z_$][\w$]*)")
56
+ _ARROW = re.compile(r"^\s*(?:export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>")
57
+
58
+
59
+ def _ext(name: str) -> str:
60
+ return PurePosixPath(name).suffix.lower()
61
+
62
+
63
+ def read_ignore(folder: Path) -> list[str]:
64
+ """Patterns from .palignore (gitignore-like: one per line, # for comments, a trailing / for folders)."""
65
+ f = folder / IGNORE_FILE
66
+ try:
67
+ lines = f.read_text(encoding="utf-8").splitlines() if f.is_file() else []
68
+ except OSError:
69
+ return []
70
+ return [x.strip() for x in lines if x.strip() and not x.strip().startswith("#")]
71
+
72
+
73
+ def ignored(rel: str, patterns: list[str], is_dir: bool = False) -> bool:
74
+ name = rel.rsplit("/", 1)[-1]
75
+ for p in patterns:
76
+ dir_only = p.endswith("/")
77
+ p = p.rstrip("/").lstrip("/")
78
+ if dir_only and not is_dir:
79
+ if any(fnmatch.fnmatch(part, p) for part in rel.split("/")[:-1]) or fnmatch.fnmatch(rel.rsplit("/", 1)[0] if "/" in rel else "", p):
80
+ return True
81
+ continue
82
+ if fnmatch.fnmatch(rel, p) or fnmatch.fnmatch(name, p) or rel.startswith(p + "/"):
83
+ return True
84
+ return False
85
+
86
+
87
+ @dataclass
88
+ class Scan:
89
+ code: dict[str, tuple[int, int]] = field(default_factory=dict) # path -> (mtime_ns, size)
90
+ docs: dict[str, tuple[int, int]] = field(default_factory=dict)
91
+ skipped: int = 0
92
+
93
+
94
+ def scan(folder: Path, patterns: list[str] | None = None) -> Scan:
95
+ """Code files and documents in the folder, leaving out hidden folders, dependencies, build output, secrets files and .palignore."""
96
+ patterns = read_ignore(folder) if patterns is None else patterns
97
+ out = Scan()
98
+ for root, dirs, files in os.walk(folder):
99
+ rel_root = Path(root).relative_to(folder).as_posix()
100
+ rel_root = "" if rel_root == "." else rel_root
101
+ dirs[:] = sorted(d for d in dirs if not d.startswith(".") and d not in SKIP_DIRS and not d.endswith(".egg-info")
102
+ and not ignored(f"{rel_root}/{d}".lstrip("/"), patterns, is_dir=True))
103
+ for name in sorted(files):
104
+ rel = f"{rel_root}/{name}".lstrip("/")
105
+ if name.startswith(".") or is_secret_file(name) or ignored(rel, patterns):
106
+ continue
107
+ ext = _ext(name)
108
+ try:
109
+ st = (Path(root) / name).stat()
110
+ except OSError:
111
+ continue
112
+ if ext.lstrip(".") in CODE_EXTENSIONS or ext in LANGUAGES:
113
+ if st.st_size <= MAX_CODE_BYTES and len(out.code) < MAX_CODE_FILES:
114
+ out.code[rel] = (st.st_mtime_ns, st.st_size)
115
+ else:
116
+ out.skipped += 1
117
+ elif ext in DOC_EXTENSIONS or ext in TEXT_DOC_EXTENSIONS:
118
+ if st.st_size <= MAX_DOC_BYTES and len(out.docs) < MAX_DOCS:
119
+ out.docs[rel] = (st.st_mtime_ns, st.st_size)
120
+ else:
121
+ out.skipped += 1
122
+ return out
123
+
124
+
125
+ # -- project map ----------------------------------------------------------------------------------------------------------------
126
+ def outline(rel: str, text: str) -> list[str]:
127
+ """The classes and functions in one file, indented by nesting (Python), as short lines."""
128
+ if rel.endswith(".py"):
129
+ try:
130
+ tree = ast.parse(text)
131
+ except (SyntaxError, ValueError):
132
+ return []
133
+ rows: list[str] = []
134
+
135
+ def walk(nodes, depth: int) -> None:
136
+ for n in nodes:
137
+ if len(rows) >= MAP_ENTRIES_PER_FILE:
138
+ return
139
+ if isinstance(n, ast.ClassDef):
140
+ rows.append(f"{' ' * depth}class {n.name}")
141
+ walk(n.body, depth + 1)
142
+ elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and not (depth and n.name.startswith("__") and n.name != "__init__"):
143
+ args = [a.arg for a in n.args.args if a.arg not in ("self", "cls")]
144
+ rows.append(f"{' ' * depth}def {n.name}({', '.join(args)})")
145
+ walk(tree.body, 0)
146
+ return rows
147
+ rows = []
148
+ for line in text.splitlines():
149
+ m = _DEF.match(line) or _ARROW.match(line)
150
+ if m:
151
+ indent = " " if line[:1] in (" ", "\t") else ""
152
+ kind = m.groupdict().get("kind") or "const"
153
+ rows.append(f"{indent}{kind} {m.group('name')}")
154
+ if len(rows) >= MAP_ENTRIES_PER_FILE:
155
+ break
156
+ return rows
157
+
158
+
159
+ def build_map(folder: Path, files: list[str]) -> str:
160
+ out = ["# Project map", "", "Every source file with its classes and functions. Ask for a file to see its code.", ""]
161
+ for rel in files:
162
+ try:
163
+ text = (folder / rel).read_text(encoding="utf-8", errors="replace")
164
+ except OSError:
165
+ continue
166
+ rows = outline(rel, text)
167
+ out.append(f"## {rel} ({text.count(chr(10)) + 1} lines)")
168
+ out += [f"- {r}" for r in rows]
169
+ out.append("")
170
+ return "\n".join(out).strip() + "\n"
171
+
172
+
173
+ def detect_tests(folder: Path) -> str:
174
+ """How this project runs its tests, when that can be told from its files."""
175
+ pkg = folder / "package.json"
176
+ if pkg.is_file():
177
+ try:
178
+ if json.loads(pkg.read_text(encoding="utf-8")).get("scripts", {}).get("test"):
179
+ return "npm test"
180
+ except (OSError, ValueError):
181
+ pass
182
+ if (folder / "Cargo.toml").is_file():
183
+ return "cargo test"
184
+ if (folder / "go.mod").is_file():
185
+ return "go test ./..."
186
+ if (folder / "pytest.ini").is_file() or (folder / "tests").is_dir() or (folder / "conftest.py").is_file():
187
+ return "python -m pytest -q"
188
+ for build in ("build.gradle", "build.gradle.kts"):
189
+ if (folder / build).is_file():
190
+ return "./gradlew test"
191
+ if (folder / "pom.xml").is_file():
192
+ return "mvn test"
193
+ return ""
194
+
195
+
196
+ # -- the code graph for the folder ----------------------------------------------------------------------------------------------
197
+ class ProjectGraph(KnowledgeGraph):
198
+ """The knowledge graph of the folder Pal works in, for every session. A project uploaded in the web chat still gets its own
199
+ graph for that session (and is removed with it); the folder's graph is never reset or pruned by a session."""
200
+
201
+ def __init__(self, project_dir: Path, cfg=None, runner=None):
202
+ super().__init__(cfg, runner)
203
+ self.project_ws = Workspace(project_dir, GRAPH_WORKSPACE)
204
+ self._uploading: set[str] = set()
205
+
206
+ @property
207
+ def enabled(self) -> bool:
208
+ return True
209
+
210
+ def _ws(self, session_id: str) -> Workspace:
211
+ own = Workspace(self.root, session_id)
212
+ return own if session_id in self._uploading or own.exists() else self.project_ws
213
+
214
+ def ingest(self, session_id: str, name: str, raw: bytes) -> dict:
215
+ self._uploading.add(session_id)
216
+ try:
217
+ return super().ingest(session_id, name, raw)
218
+ finally:
219
+ self._uploading.discard(session_id)
220
+
221
+ def reset(self, session_id: str) -> None:
222
+ Workspace(self.root, session_id).reset()
223
+
224
+
225
+ # -- the whole setup ------------------------------------------------------------------------------------------------------------
226
+ @dataclass
227
+ class DocResult:
228
+ path: str
229
+ markdown: str = "" # where the markdown file is
230
+ tokens: int = 0
231
+ converted: bool = False
232
+ error: str = ""
233
+
234
+
235
+ @dataclass
236
+ class Report:
237
+ code_files: int = 0
238
+ source_tokens: int = 0
239
+ languages: list[str] = field(default_factory=list)
240
+ docs: list[DocResult] = field(default_factory=list)
241
+ map_tokens: int = 0
242
+ tests: str = ""
243
+ graph: str = "not built" # "ready: N nodes", "building", "needs graphify", "failed: ...", "no code"
244
+ seconds: float = 0.0
245
+ skipped: int = 0
246
+
247
+ @property
248
+ def converted(self) -> list[DocResult]:
249
+ return [d for d in self.docs if d.converted and not d.error]
250
+
251
+
252
+ class Project:
253
+ def __init__(self, folder: Path, home: Path | None = None, runner=None, graph_cfg=None):
254
+ self.folder = Path(folder).resolve()
255
+ self.root = (home or pal_home()) / "projects" / project_id(self.folder)
256
+ self.markdown_dir = self.root / "markdown"
257
+ self.manifest_path = self.root / "project.json"
258
+ self.root.mkdir(parents=True, exist_ok=True)
259
+ self.graph = ProjectGraph(self.root / "graph", graph_cfg or {"enabled": True, "dir": str(self.root / "graph-uploads")}, runner)
260
+ self.report = Report()
261
+ self.texts: dict[str, str] = {} # name -> markdown, what is indexed for every session
262
+ self._thread: threading.Thread | None = None
263
+ self._lock = threading.Lock()
264
+
265
+ # -- manifest -------------------------------------------------------------------------------------------------------------
266
+ def _manifest(self) -> dict:
267
+ try:
268
+ return json.loads(self.manifest_path.read_text(encoding="utf-8"))
269
+ except (OSError, ValueError):
270
+ return {}
271
+
272
+ def _save_manifest(self, data: dict) -> None:
273
+ tmp = self.manifest_path.with_suffix(".tmp")
274
+ tmp.write_text(json.dumps(data, indent=1), encoding="utf-8")
275
+ tmp.replace(self.manifest_path)
276
+
277
+ # -- steps ----------------------------------------------------------------------------------------------------------------
278
+ def prepare(self, graph_in_background: bool = True, progress=None) -> Report:
279
+ """Scan, convert documents, write the project map, and build (or update) the code graph. Returns the report."""
280
+ t0 = time.perf_counter()
281
+ say = progress or (lambda _msg: None)
282
+ found = scan(self.folder)
283
+ old = self._manifest()
284
+ r = Report(code_files=len(found.code), skipped=found.skipped, tests=detect_tests(self.folder))
285
+ r.source_tokens = sum(size // 4 for _, size in found.code.values())
286
+ langs = Counter(LANGUAGES[_ext(p)] for p in found.code if _ext(p) in LANGUAGES)
287
+ r.languages = [name for name, _ in langs.most_common(4)]
288
+
289
+ say(f"documents: {len(found.docs)}")
290
+ old_docs = old.get("docs", {})
291
+ docs_state = {}
292
+ self.texts = {}
293
+ for rel, stamp in found.docs.items():
294
+ res, state = self._document(rel, stamp, old_docs.get(rel))
295
+ r.docs.append(res)
296
+ docs_state[rel] = state
297
+ for rel in set(old_docs) - set(found.docs): # a document that was removed: drop its markdown
298
+ md = old_docs[rel].get("markdown")
299
+ if md:
300
+ Path(md).unlink(missing_ok=True)
301
+
302
+ say(f"project map: {len(found.code)} files")
303
+ if found.code:
304
+ text = build_map(self.folder, sorted(found.code))
305
+ (self.root / "project-map.md").write_text(text, encoding="utf-8")
306
+ r.map_tokens = estimate_tokens(text)
307
+ self.texts["project-map.md"] = text
308
+ self.report = r
309
+ self._save_manifest({"folder": str(self.folder), "docs": docs_state, "code": {k: list(v) for k, v in found.code.items()},
310
+ "graph": old.get("graph", {}), "updated": time.time()})
311
+
312
+ if not found.code:
313
+ r.graph = "no code"
314
+ elif not self.graph.available():
315
+ r.graph = "needs graphify"
316
+ else:
317
+ changed = old.get("code") != {k: list(v) for k, v in found.code.items()} or not self.graph.project_ws.exists()
318
+ if not changed:
319
+ r.graph = f"ready: {old.get('graph', {}).get('nodes', 0):,} nodes"
320
+ else:
321
+ r.graph = "building"
322
+ if graph_in_background:
323
+ self._thread = threading.Thread(target=self._build_graph, args=(found,), daemon=True, name="pal-graph")
324
+ self._thread.start()
325
+ else:
326
+ self._build_graph(found)
327
+ r.seconds = time.perf_counter() - t0
328
+ return r
329
+
330
+ def _document(self, rel: str, stamp: tuple[int, int], before: dict | None) -> tuple[DocResult, dict]:
331
+ src = self.folder / rel
332
+ ext = _ext(rel)
333
+ md_path = self.markdown_dir / (rel + ".md")
334
+ if before and tuple(before.get("stamp", ())) == tuple(stamp) and before.get("error"):
335
+ return DocResult(rel, error=before["error"]), before # unchanged since it failed: not tried again
336
+ if before and tuple(before.get("stamp", ())) == tuple(stamp) and (ext in TEXT_DOC_EXTENSIONS or md_path.is_file()):
337
+ try:
338
+ text = md_path.read_text(encoding="utf-8") if ext not in TEXT_DOC_EXTENSIONS else src.read_text(encoding="utf-8", errors="replace")
339
+ self.texts[rel] = text
340
+ return DocResult(rel, str(md_path) if ext not in TEXT_DOC_EXTENSIONS else "", estimate_tokens(text), ext not in TEXT_DOC_EXTENSIONS), before
341
+ except OSError:
342
+ pass
343
+ if ext in TEXT_DOC_EXTENSIONS:
344
+ try:
345
+ text = src.read_text(encoding="utf-8", errors="replace")
346
+ except OSError as e:
347
+ return DocResult(rel, error=str(e.strerror or e)), {"stamp": list(stamp), "error": "unreadable"}
348
+ self.texts[rel] = text
349
+ return DocResult(rel, "", estimate_tokens(text)), {"stamp": list(stamp)}
350
+ from pal.input_converters import convert_to_markdown
351
+ try:
352
+ text = convert_to_markdown(src.name, src.read_bytes()).text
353
+ except Exception as e: # noqa: BLE001 (damaged file, a scan with no text, a converter that is missing)
354
+ return DocResult(rel, error=str(e)[:160]), {"stamp": list(stamp), "error": str(e)[:160]}
355
+ md_path.parent.mkdir(parents=True, exist_ok=True)
356
+ md_path.write_text(text, encoding="utf-8")
357
+ self.texts[rel] = text
358
+ return DocResult(rel, str(md_path), estimate_tokens(text), True), {"stamp": list(stamp), "markdown": str(md_path)}
359
+
360
+ def _build_graph(self, found: Scan) -> None:
361
+ with self._lock:
362
+ ws = self.graph.project_ws
363
+ try:
364
+ ws.src.mkdir(parents=True, exist_ok=True)
365
+ keep = set()
366
+ for rel in found.code: # copy what changed; the folder itself is never touched
367
+ if _ext(rel).lstrip(".") not in CODE_EXTENSIONS:
368
+ continue
369
+ src, dst = self.folder / rel, ws.src / rel
370
+ keep.add(dst)
371
+ try:
372
+ if not dst.is_file() or dst.stat().st_size != src.stat().st_size or dst.stat().st_mtime_ns < src.stat().st_mtime_ns:
373
+ dst.parent.mkdir(parents=True, exist_ok=True)
374
+ shutil.copy2(src, dst)
375
+ except OSError:
376
+ continue
377
+ for p in list(ws.src.rglob("*")): # files removed from the folder
378
+ if p.is_file() and "graphify-out" not in p.parts and p not in keep:
379
+ p.unlink(missing_ok=True)
380
+ info = builder.build(ws, self.folder.name, self.graph.runner)
381
+ self.report.graph = f"ready: {info.nodes:,} nodes"
382
+ data = self._manifest()
383
+ data["graph"] = {"nodes": info.nodes, "edges": info.edges, "files": info.files, "built": time.time()}
384
+ self._save_manifest(data)
385
+ except Exception as e: # noqa: BLE001 (the graph is a help, never a reason for the session to fail)
386
+ self.report.graph = f"failed: {str(e)[:120]}"
387
+
388
+ def wait(self, timeout: float | None = None) -> None:
389
+ if self._thread is not None:
390
+ self._thread.join(timeout)
391
+
392
+ # -- for the model --------------------------------------------------------------------------------------------------------
393
+ def notes(self) -> str:
394
+ """PAL.md in the folder: the project's own instructions for Pal (written by you, or started with /init)."""
395
+ f = self.folder / NOTES_FILE
396
+ try:
397
+ return f.read_text(encoding="utf-8", errors="replace").strip()[:MAX_NOTES_CHARS] if f.is_file() else ""
398
+ except OSError:
399
+ return ""
400
+
401
+ def brief(self) -> str:
402
+ r = self.report
403
+ parts = [f"Project: {self.folder.name}, {r.code_files} source files" + (f" ({', '.join(r.languages)})" if r.languages else "")]
404
+ if r.tests:
405
+ parts.append(f"Tests run with: {r.tests}")
406
+ if self.texts:
407
+ parts.append("Indexed for you (passages that match each request are added): " + ", ".join(sorted(self.texts)[:12])
408
+ + (f" and {len(self.texts) - 12} more" if len(self.texts) > 12 else ""))
409
+ notes = self.notes()
410
+ return "[" + ". ".join(parts) + ".]" + (f"\n\nProject instructions ({NOTES_FILE}), follow them:\n{notes}" if notes else "")
411
+
412
+ def documents(self) -> list[tuple[str, str]]:
413
+ return sorted(self.texts.items())
pal/terminal/review.py ADDED
@@ -0,0 +1,65 @@
1
+ """/review: the pending changes as a page in your browser, in Pal's colours. A plain file on this computer: no server, nothing online.
2
+
3
+ Every text from the model is escaped before it goes in the page, and the page carries a strict content security policy (no scripts at all),
4
+ so a change that contains HTML is shown as text and cannot run.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import html
9
+ import time
10
+ import webbrowser
11
+ from pathlib import Path
12
+
13
+ from pal.paths import pal_home
14
+ from pal.terminal.workflow import Workflow
15
+
16
+ CSS = """
17
+ :root{color-scheme:dark;--bg:#000;--panel:#121212;--line:#262626;--ink:#e8e8e8;--muted:#8a8a8a;--red:#E50914;--red-hi:#FF4D57;--add:#1d2b1f;--del:#2b1214}
18
+ *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 Inter,system-ui,sans-serif}
19
+ header{position:sticky;top:0;background:#0a0a0a;border-bottom:1px solid var(--line);padding:14px 24px;display:flex;gap:16px;align-items:baseline;flex-wrap:wrap}
20
+ header h1{margin:0;font-size:18px}header h1 b{color:var(--red)}header span{color:var(--muted);font-size:13px}
21
+ .wrap{display:grid;grid-template-columns:260px minmax(0,1fr);gap:0}
22
+ nav{border-right:1px solid var(--line);padding:16px 12px;position:sticky;top:56px;align-self:start;max-height:calc(100vh - 56px);overflow:auto}
23
+ nav a{display:block;color:var(--ink);text-decoration:none;font:13px ui-monospace,Menlo,Consolas,monospace;padding:5px 8px;border-radius:6px;overflow-wrap:anywhere}
24
+ nav a:hover{background:var(--panel)}nav a i{font-style:normal;color:var(--red-hi);margin-right:6px}
25
+ main{padding:16px 24px 60px;min-width:0}
26
+ section{border:1px solid var(--line);border-radius:10px;margin:0 0 20px;overflow:hidden}
27
+ section h2{margin:0;padding:10px 14px;background:var(--panel);font:600 13px ui-monospace,Menlo,Consolas,monospace;border-bottom:1px solid var(--line)}
28
+ section h2 small{color:var(--muted);font-weight:400;margin-left:8px}
29
+ .find{padding:8px 14px;border-bottom:1px solid var(--line);font-size:13px}.find b{color:#fff;background:var(--red);padding:0 6px;border-radius:4px;margin-right:6px;font-size:11px}
30
+ pre{margin:0;padding:8px 0;overflow-x:auto;font:12.5px/1.55 ui-monospace,Menlo,Consolas,monospace}
31
+ pre span{display:block;padding:0 14px;white-space:pre}.a{background:var(--add);color:#cfe9d3}.d{background:var(--del);color:#ffc2c6}.h{color:var(--muted)}.c{color:#a8a8a8}
32
+ footer{color:var(--muted);font-size:13px;padding:0 24px 24px}code{color:var(--ink)}
33
+ @media (max-width:800px){.wrap{grid-template-columns:1fr}nav{position:static;max-height:none;border-right:0;border-bottom:1px solid var(--line)}}
34
+ """
35
+
36
+
37
+ def render(wf: Workflow) -> str:
38
+ diffs = wf.diffs()
39
+ esc = html.escape
40
+ nav = "".join(f'<a href="#f{i}"><i>{"+" if new else "~"}</i>{esc(path)}</a>' for i, (path, new, _, _) in enumerate(diffs))
41
+ parts = []
42
+ for i, (path, new, lines, findings) in enumerate(diffs):
43
+ body = "".join(f'<span class="{ {"add": "a", "del": "d", "hunk": "h"}.get(tag, "c") }">{esc(("+ " if tag == "add" else "- " if tag == "del" else "" if tag == "hunk" else " ") + text)}</span>' for tag, text in lines)
44
+ found = "".join(f'<div class="find"><b>{esc(f.severity.upper())}</b>{esc(f.title)} (line {f.line}): {esc(f.why)}</div>' for f in findings)
45
+ parts.append(f'<section id="f{i}"><h2>{esc(path)}<small>{"new file" if new else "changed"}</small></h2>{found}<pre>{body}</pre></section>')
46
+ cmds = wf.plan.b.commands
47
+ empty = "<p>Nothing is pending.</p>" if not diffs else ""
48
+ return f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
49
+ <meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'; img-src data:">
50
+ <title>Pal review: {esc(wf.plan.name)}</title><style>{CSS}</style></head><body>
51
+ <header><h1><b>Pal</b> review</h1><span>plan {esc(wf.plan.name)} · branch {esc(wf.plan.branch)} · {len(diffs)} file{'s' if len(diffs) != 1 else ''} pending · {time.strftime('%H:%M')}</span></header>
52
+ <div class="wrap"><nav>{nav or '<span>No files</span>'}</nav><main>{empty}{''.join(parts)}</main></div>
53
+ <footer>Back in the terminal: <code>/apply</code> writes these changes{', then runs ' + esc(', '.join(cmds)) + ' if you approve' if cmds else ''}; <code>/reject &lt;file&gt;</code> drops one; <code>/rewind</code> goes back a step.</footer>
54
+ </body></html>"""
55
+
56
+
57
+ def open_review(wf: Workflow, open_browser=webbrowser.open) -> Path:
58
+ out = pal_home() / "review" / f"{wf.plan.name}-{wf.plan.branch}.html"
59
+ out.parent.mkdir(parents=True, exist_ok=True)
60
+ out.write_text(render(wf), encoding="utf-8")
61
+ try:
62
+ open_browser(out.resolve().as_uri())
63
+ except Exception: # noqa: BLE001 (no browser: the path is shown instead)
64
+ pass
65
+ return out