footprinter-cli 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. footprinter/__init__.py +8 -0
  2. footprinter/access.py +444 -0
  3. footprinter/api/__init__.py +1 -0
  4. footprinter/api/db.py +61 -0
  5. footprinter/api/entities.py +250 -0
  6. footprinter/api/search.py +47 -0
  7. footprinter/api/semantic.py +33 -0
  8. footprinter/api/server.py +66 -0
  9. footprinter/api/status.py +15 -0
  10. footprinter/bundled/__init__.py +0 -0
  11. footprinter/bundled/config.example.yaml +161 -0
  12. footprinter/bundled/patterns/context_patterns.yaml +18 -0
  13. footprinter/bundled/patterns/extensions.yaml +283 -0
  14. footprinter/bundled/patterns/filename_patterns.yaml +61 -0
  15. footprinter/bundled/patterns/mime_mappings.yaml +68 -0
  16. footprinter/bundled/patterns/salesforce_rules.yaml +84 -0
  17. footprinter/bundled/patterns/security_patterns.yaml +27 -0
  18. footprinter/cli/__init__.py +128 -0
  19. footprinter/cli/__main__.py +6 -0
  20. footprinter/cli/_common.py +332 -0
  21. footprinter/cli/_policy_helpers.py +646 -0
  22. footprinter/cli/_prompt.py +220 -0
  23. footprinter/cli/api_cmd.py +32 -0
  24. footprinter/cli/connect.py +591 -0
  25. footprinter/cli/data.py +879 -0
  26. footprinter/cli/delete.py +128 -0
  27. footprinter/cli/ingest.py +579 -0
  28. footprinter/cli/mcp_cmd.py +750 -0
  29. footprinter/cli/mcp_setup.py +306 -0
  30. footprinter/cli/search.py +393 -0
  31. footprinter/cli/search_cmd.py +69 -0
  32. footprinter/cli/setup.py +1836 -0
  33. footprinter/cli/status.py +729 -0
  34. footprinter/cli/status_cmd.py +104 -0
  35. footprinter/cli/upsert.py +794 -0
  36. footprinter/cli/vectorize_cmd.py +215 -0
  37. footprinter/cli/view.py +322 -0
  38. footprinter/connectors/__init__.py +171 -0
  39. footprinter/connectors/config_utils.py +141 -0
  40. footprinter/db/__init__.py +37 -0
  41. footprinter/db/browser.py +198 -0
  42. footprinter/db/chats.py +610 -0
  43. footprinter/db/clients.py +307 -0
  44. footprinter/db/emails.py +279 -0
  45. footprinter/db/files.py +741 -0
  46. footprinter/db/folders.py +659 -0
  47. footprinter/db/messages.py +192 -0
  48. footprinter/db/policies.py +151 -0
  49. footprinter/db/projects.py +673 -0
  50. footprinter/db/search.py +573 -0
  51. footprinter/db/sql_utils.py +168 -0
  52. footprinter/db/status.py +320 -0
  53. footprinter/db/uploads.py +70 -0
  54. footprinter/ingest/__init__.py +0 -0
  55. footprinter/ingest/adapters/__init__.py +33 -0
  56. footprinter/ingest/adapters/browser.py +54 -0
  57. footprinter/ingest/adapters/chat.py +57 -0
  58. footprinter/ingest/adapters/ingest.py +146 -0
  59. footprinter/ingest/adapters/local_files.py +68 -0
  60. footprinter/ingest/adapters/local_folders.py +52 -0
  61. footprinter/ingest/adapters/protocol.py +174 -0
  62. footprinter/ingest/browser_indexer.py +216 -0
  63. footprinter/ingest/chat_dedup.py +156 -0
  64. footprinter/ingest/chat_indexer.py +515 -0
  65. footprinter/ingest/chat_parsers/__init__.py +8 -0
  66. footprinter/ingest/chat_parsers/chatgpt_parser.py +229 -0
  67. footprinter/ingest/chat_parsers/claude_parser.py +161 -0
  68. footprinter/ingest/cli.py +827 -0
  69. footprinter/ingest/content_extractors.py +117 -0
  70. footprinter/ingest/database.py +36 -0
  71. footprinter/ingest/db/__init__.py +1 -0
  72. footprinter/ingest/db/connector_schema.py +47 -0
  73. footprinter/ingest/db/migration.py +328 -0
  74. footprinter/ingest/db/schema.py +1043 -0
  75. footprinter/ingest/db/security.py +6 -0
  76. footprinter/ingest/file_indexer.py +261 -0
  77. footprinter/ingest/file_scanner.py +277 -0
  78. footprinter/ingest/folder_indexer.py +226 -0
  79. footprinter/ingest/full_content_extractor.py +321 -0
  80. footprinter/ingest/orchestrator.py +125 -0
  81. footprinter/ingest/pipe_runner.py +217 -0
  82. footprinter/ingest/processing.py +165 -0
  83. footprinter/ingest/registry.py +201 -0
  84. footprinter/ingest/run_record.py +91 -0
  85. footprinter/ingest/status.py +346 -0
  86. footprinter/mcp/__init__.py +0 -0
  87. footprinter/mcp/__main__.py +5 -0
  88. footprinter/mcp/db.py +57 -0
  89. footprinter/mcp/errors.py +102 -0
  90. footprinter/mcp/extraction.py +226 -0
  91. footprinter/mcp/server.py +39 -0
  92. footprinter/mcp/tools/__init__.py +0 -0
  93. footprinter/mcp/tools/navigation.py +70 -0
  94. footprinter/mcp/tools/read.py +75 -0
  95. footprinter/mcp/tools/search.py +158 -0
  96. footprinter/mcp/tools/semantic.py +79 -0
  97. footprinter/mcp/tools/status.py +15 -0
  98. footprinter/paths.py +91 -0
  99. footprinter/permissions.py +1160 -0
  100. footprinter/semantic/__init__.py +13 -0
  101. footprinter/semantic/chunking.py +52 -0
  102. footprinter/semantic/embeddings.py +23 -0
  103. footprinter/semantic/hybrid_search.py +273 -0
  104. footprinter/semantic/vector_store.py +471 -0
  105. footprinter/services/__init__.py +49 -0
  106. footprinter/services/access_service.py +342 -0
  107. footprinter/services/chat_service.py +85 -0
  108. footprinter/services/client_service.py +267 -0
  109. footprinter/services/content_service.py +181 -0
  110. footprinter/services/email_service.py +89 -0
  111. footprinter/services/file_service.py +83 -0
  112. footprinter/services/folder_service.py +122 -0
  113. footprinter/services/includes.py +19 -0
  114. footprinter/services/ingest_service.py +231 -0
  115. footprinter/services/project_service.py +262 -0
  116. footprinter/services/roles.py +25 -0
  117. footprinter/services/search_service.py +177 -0
  118. footprinter/services/semantic_service.py +360 -0
  119. footprinter/services/status_service.py +18 -0
  120. footprinter/services/visit_service.py +65 -0
  121. footprinter/source_registry.py +194 -0
  122. footprinter/utils/__init__.py +7 -0
  123. footprinter/utils/hash_utils.py +59 -0
  124. footprinter/utils/logging_config.py +68 -0
  125. footprinter/utils/mime.py +30 -0
  126. footprinter/utils/text.py +6 -0
  127. footprinter/utils/time.py +11 -0
  128. footprinter/visibility.py +1272 -0
  129. footprinter_cli-1.0.0.dist-info/LICENSE +21 -0
  130. footprinter_cli-1.0.0.dist-info/METADATA +229 -0
  131. footprinter_cli-1.0.0.dist-info/RECORD +134 -0
  132. footprinter_cli-1.0.0.dist-info/WHEEL +5 -0
  133. footprinter_cli-1.0.0.dist-info/entry_points.txt +2 -0
  134. footprinter_cli-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,192 @@
1
+ """Cross-chat message queries.
2
+
3
+ Provides list, get, and search functions for messages across all chats.
4
+ All functions take sqlite3.Connection and return plain dicts.
5
+ """
6
+
7
+ import sqlite3
8
+ from typing import Optional
9
+
10
+ from footprinter.db.sql_utils import paginate, paginated_response
11
+
12
+
13
+ def list_messages(
14
+ conn: sqlite3.Connection,
15
+ *,
16
+ role: Optional[str] = None,
17
+ account: Optional[str] = None,
18
+ chat_id: Optional[int] = None,
19
+ limit: int = 50,
20
+ page: int = 1,
21
+ ) -> dict:
22
+ """List messages across chats with optional filters and pagination.
23
+
24
+ Parameters
25
+ ----------
26
+ conn : sqlite3.Connection
27
+ role : optional role filter (e.g. 'user', 'assistant')
28
+ account : optional account filter (e.g. 'claude', 'chatgpt')
29
+ chat_id : optional chat ID to filter to a single chat
30
+ limit : max rows per page
31
+ page : 1-based page number
32
+
33
+ Returns
34
+ -------
35
+ dict with keys: messages, pagination
36
+ """
37
+ conditions: list[str] = []
38
+ params: list = []
39
+
40
+ if role:
41
+ conditions.append("message.role = ?")
42
+ params.append(role)
43
+
44
+ if account:
45
+ conditions.append("chat.account = ?")
46
+ params.append(account)
47
+
48
+ if chat_id is not None:
49
+ conditions.append("message.chat_id = ?")
50
+ params.append(chat_id)
51
+
52
+ where = "WHERE " + " AND ".join(conditions) if conditions else ""
53
+
54
+ count_sql = f"""
55
+ SELECT COUNT(*)
56
+ FROM messages message
57
+ JOIN chats chat ON message.chat_id = chat.id
58
+ {where}
59
+ """
60
+ fetch_sql = f"""
61
+ SELECT message.id, message.chat_id, message.message_id, message.role, message.content, message.created_at,
62
+ chat.title AS chat_title, chat.account AS chat_account,
63
+ message.mcp_view, message.mcp_read
64
+ FROM messages message
65
+ JOIN chats chat ON message.chat_id = chat.id
66
+ {where}
67
+ ORDER BY message.id DESC
68
+ LIMIT ? OFFSET ?
69
+ """
70
+
71
+ rows, pagination = paginate(conn, count_sql, fetch_sql, params, page=page, limit=limit)
72
+
73
+ messages = [
74
+ {
75
+ "id": r["id"],
76
+ "chat_id": r["chat_id"],
77
+ "message_id": r["message_id"],
78
+ "role": r["role"],
79
+ "content": r["content"],
80
+ "created_at": r["created_at"],
81
+ "chat_title": r["chat_title"],
82
+ "chat_account": r["chat_account"],
83
+ "mcp_view": r["mcp_view"],
84
+ "mcp_read": r["mcp_read"],
85
+ }
86
+ for r in rows
87
+ ]
88
+
89
+ return paginated_response("messages", messages, pagination)
90
+
91
+
92
+ def get_message(conn: sqlite3.Connection, message_id: int) -> Optional[dict]:
93
+ """Get a single message with chat context.
94
+
95
+ Parameters
96
+ ----------
97
+ conn : sqlite3.Connection
98
+ message_id : internal integer ID
99
+
100
+ Returns
101
+ -------
102
+ dict with message fields and chat context, or None if not found
103
+ """
104
+ cursor = conn.execute(
105
+ """
106
+ SELECT message.id, message.chat_id, message.message_id, message.role, message.content, message.created_at,
107
+ chat.title AS chat_title, chat.account AS chat_account,
108
+ message.mcp_view, message.mcp_read
109
+ FROM messages message
110
+ JOIN chats chat ON message.chat_id = chat.id
111
+ WHERE message.id = ?
112
+ """,
113
+ (message_id,),
114
+ )
115
+ row = cursor.fetchone()
116
+ if not row:
117
+ return None
118
+
119
+ return {
120
+ "id": row["id"],
121
+ "chat_id": row["chat_id"],
122
+ "message_id": row["message_id"],
123
+ "role": row["role"],
124
+ "content": row["content"],
125
+ "created_at": row["created_at"],
126
+ "chat_title": row["chat_title"],
127
+ "chat_account": row["chat_account"],
128
+ "mcp_view": row["mcp_view"],
129
+ "mcp_read": row["mcp_read"],
130
+ }
131
+
132
+
133
+ def search_messages(
134
+ conn: sqlite3.Connection,
135
+ query: str,
136
+ *,
137
+ limit: int = 50,
138
+ page: int = 1,
139
+ ) -> dict:
140
+ """Search message content across all chats.
141
+
142
+ Parameters
143
+ ----------
144
+ conn : sqlite3.Connection
145
+ query : search term
146
+ limit : max results per page
147
+ page : 1-based page number
148
+
149
+ Returns
150
+ -------
151
+ dict with keys: results, pagination
152
+ """
153
+ if len(query) < 2:
154
+ return paginated_response("results", [], {"page": page, "limit": limit, "total": 0, "total_pages": 1})
155
+
156
+ query_param = f"%{query}%"
157
+ count_sql = """
158
+ SELECT COUNT(*)
159
+ FROM messages message
160
+ JOIN chats chat ON message.chat_id = chat.id
161
+ WHERE message.content LIKE ?
162
+ """
163
+ fetch_sql = """
164
+ SELECT message.id, message.chat_id, message.message_id, message.role, message.content, message.created_at,
165
+ chat.title AS chat_title, chat.account AS chat_account,
166
+ message.mcp_view, message.mcp_read
167
+ FROM messages message
168
+ JOIN chats chat ON message.chat_id = chat.id
169
+ WHERE message.content LIKE ?
170
+ ORDER BY message.id DESC
171
+ LIMIT ? OFFSET ?
172
+ """
173
+
174
+ rows, pagination = paginate(conn, count_sql, fetch_sql, [query_param], page=page, limit=limit)
175
+
176
+ results = [
177
+ {
178
+ "id": r["id"],
179
+ "chat_id": r["chat_id"],
180
+ "message_id": r["message_id"],
181
+ "role": r["role"],
182
+ "content": r["content"],
183
+ "created_at": r["created_at"],
184
+ "chat_title": r["chat_title"],
185
+ "chat_account": r["chat_account"],
186
+ "mcp_view": r["mcp_view"],
187
+ "mcp_read": r["mcp_read"],
188
+ }
189
+ for r in rows
190
+ ]
191
+
192
+ return paginated_response("results", results, pagination)
@@ -0,0 +1,151 @@
1
+ """Access control policy CRUD — visibility and permission layers."""
2
+
3
+ import sqlite3
4
+
5
+ PERMISSION_SETTINGS = frozenset({"allow", "deny"})
6
+ VISIBILITY_SETTINGS = frozenset({"visible", "opaque", "hidden"})
7
+
8
+ SCOPE_PREFIXES = frozenset({"source", "account", "folder", "project", "client", "file", "email", "chat"})
9
+ VALID_SOURCE_TYPES = frozenset({"files", "emails", "chats", "folders", "browser", "projects", "clients"})
10
+ _ID_PREFIXES = frozenset({"project", "client", "file", "email", "chat"})
11
+
12
+
13
+ def is_folder_path_scope(scope: str) -> bool:
14
+ """True if scope is a folder path prefix (not a numeric folder ID)."""
15
+ suffix = scope[len("folder:") :]
16
+ return not suffix.isdigit()
17
+
18
+
19
+ def validate_scope(scope: str) -> None:
20
+ """Raise ValueError if *scope* is not a recognised scope pattern."""
21
+ if scope == "global":
22
+ return
23
+ if ":" in scope:
24
+ prefix, value = scope.split(":", 1)
25
+ if prefix not in SCOPE_PREFIXES:
26
+ raise ValueError(f"Invalid scope: {scope!r}. Unknown prefix {prefix!r}.")
27
+ if not value or value.isspace():
28
+ raise ValueError(f"Invalid scope: {scope!r}. Value after '{prefix}:' must not be empty.")
29
+ if prefix == "source" and value not in VALID_SOURCE_TYPES:
30
+ raise ValueError(f"Invalid scope: {scope!r}. Valid source types: {', '.join(sorted(VALID_SOURCE_TYPES))}")
31
+ if prefix in _ID_PREFIXES:
32
+ try:
33
+ int(value)
34
+ except ValueError:
35
+ raise ValueError(f"Invalid scope: {scope!r}. '{prefix}:' requires a numeric ID.") from None
36
+ return
37
+ raise ValueError(f"Invalid scope: {scope!r}. Expected 'global' or 'prefix:value'.")
38
+
39
+
40
+ # ---------------------------------------------------------------------------
41
+ # Visibility policies
42
+ # ---------------------------------------------------------------------------
43
+
44
+
45
+ def list_visibility_policies(conn: sqlite3.Connection) -> list[dict]:
46
+ """Return all visibility policies as plain dicts."""
47
+ rows = conn.execute("SELECT scope, setting, updated_at FROM visibility_policies ORDER BY scope").fetchall()
48
+ return [{"scope": r["scope"], "setting": r["setting"], "updated_at": r["updated_at"]} for r in rows]
49
+
50
+
51
+ def set_visibility_policy(conn: sqlite3.Connection, scope: str, setting: str) -> bool:
52
+ """Insert or update a visibility policy. Returns True."""
53
+ validate_scope(scope)
54
+ if setting not in VISIBILITY_SETTINGS:
55
+ raise ValueError(f"Invalid visibility setting: {setting}. Valid: {', '.join(sorted(VISIBILITY_SETTINGS))}")
56
+ conn.execute(
57
+ "INSERT OR REPLACE INTO visibility_policies (scope, setting, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
58
+ (scope, setting),
59
+ )
60
+ conn.commit()
61
+ return True
62
+
63
+
64
+ def delete_visibility_policy(conn: sqlite3.Connection, scope: str) -> bool:
65
+ """Delete a visibility policy. Returns True if a row was removed."""
66
+ cur = conn.cursor()
67
+ cur.execute("DELETE FROM visibility_policies WHERE scope = ?", (scope,))
68
+ deleted = cur.rowcount > 0
69
+ conn.commit()
70
+ return deleted
71
+
72
+
73
+ def clear_visibility_policies(conn: sqlite3.Connection) -> int:
74
+ """Delete all visibility policies. Returns count of rows removed."""
75
+ cur = conn.cursor()
76
+ cur.execute("DELETE FROM visibility_policies")
77
+ count = cur.rowcount
78
+ conn.commit()
79
+ return count
80
+
81
+
82
+ def seed_visibility_defaults(conn: sqlite3.Connection) -> bool:
83
+ """Seed ``global=visible`` into visibility_policies. Idempotent."""
84
+ cur = conn.cursor()
85
+ cur.execute("INSERT OR IGNORE INTO visibility_policies (scope, setting) VALUES ('global', 'visible')")
86
+ seeded = cur.rowcount > 0
87
+ conn.commit()
88
+ return seeded
89
+
90
+
91
+ # ---------------------------------------------------------------------------
92
+ # Permission policies
93
+ # ---------------------------------------------------------------------------
94
+
95
+
96
+ def list_permission_policies(conn: sqlite3.Connection) -> list[dict]:
97
+ """Return all permission policies as plain dicts."""
98
+ rows = conn.execute("SELECT scope, setting, updated_at FROM permission_policies ORDER BY scope").fetchall()
99
+ return [{"scope": r["scope"], "setting": r["setting"], "updated_at": r["updated_at"]} for r in rows]
100
+
101
+
102
+ def set_permission_policy(conn: sqlite3.Connection, scope: str, setting: str) -> bool:
103
+ """Insert or update a permission policy. Returns True."""
104
+ validate_scope(scope)
105
+ if setting not in PERMISSION_SETTINGS:
106
+ raise ValueError(f"Invalid permission setting: {setting}. Valid: {', '.join(sorted(PERMISSION_SETTINGS))}")
107
+ conn.execute(
108
+ "INSERT OR REPLACE INTO permission_policies (scope, setting, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
109
+ (scope, setting),
110
+ )
111
+ conn.commit()
112
+ return True
113
+
114
+
115
+ def delete_permission_policy(conn: sqlite3.Connection, scope: str) -> bool:
116
+ """Delete a permission policy. Returns True if a row was removed."""
117
+ cur = conn.cursor()
118
+ cur.execute("DELETE FROM permission_policies WHERE scope = ?", (scope,))
119
+ deleted = cur.rowcount > 0
120
+ conn.commit()
121
+ return deleted
122
+
123
+
124
+ def clear_permission_policies(conn: sqlite3.Connection) -> int:
125
+ """Delete all permission policies. Returns count of rows removed."""
126
+ cur = conn.cursor()
127
+ cur.execute("DELETE FROM permission_policies")
128
+ count = cur.rowcount
129
+ conn.commit()
130
+ return count
131
+
132
+
133
+ def seed_permission_defaults(conn: sqlite3.Connection) -> bool:
134
+ """Seed ``global=allow`` into permission_policies. Idempotent."""
135
+ cur = conn.cursor()
136
+ cur.execute("INSERT OR IGNORE INTO permission_policies (scope, setting) VALUES ('global', 'allow')")
137
+ seeded = cur.rowcount > 0
138
+ conn.commit()
139
+ return seeded
140
+
141
+
142
+ # ---------------------------------------------------------------------------
143
+ # Combined seed
144
+ # ---------------------------------------------------------------------------
145
+
146
+
147
+ def seed_access_policies(conn: sqlite3.Connection) -> dict:
148
+ """Seed both visibility and permission defaults. Returns status dict."""
149
+ vis = seed_visibility_defaults(conn)
150
+ perm = seed_permission_defaults(conn)
151
+ return {"visibility_seeded": vis, "permission_seeded": perm}