looker-sync 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looker_sync-0.1.0/.claude/skills/codebase-map/SKILL.md +25 -0
- looker_sync-0.1.0/.claude/skills/repo-maintenance/SKILL.md +47 -0
- looker_sync-0.1.0/.gitignore +6 -0
- looker_sync-0.1.0/LICENSE +21 -0
- looker_sync-0.1.0/PKG-INFO +59 -0
- looker_sync-0.1.0/README.md +43 -0
- looker_sync-0.1.0/docs/README.md +47 -0
- looker_sync-0.1.0/pyproject.toml +36 -0
- looker_sync-0.1.0/skills/looker-sync/SKILL.md +101 -0
- looker_sync-0.1.0/src/looker_sync/__init__.py +0 -0
- looker_sync-0.1.0/src/looker_sync/__main__.py +4 -0
- looker_sync-0.1.0/src/looker_sync/cli.py +740 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: codebase-map
|
|
3
|
+
description: Where behaviour lives in looker-sync: which file owns the sync rules, the guards, the Looker API calls and the tests. Use before hunting for where something is implemented or before changing a guard.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Where things live
|
|
7
|
+
|
|
8
|
+
Open the one page for your area, then go straight to the file. Line numbers are a
|
|
9
|
+
starting point, not an address; confirm by what the code says.
|
|
10
|
+
|
|
11
|
+
<!-- BEGIN GENERATED INDEX -- edit the pages, not this block -->
|
|
12
|
+
|
|
13
|
+
## Where things live
|
|
14
|
+
|
|
15
|
+
One page per area of the system. Read before going looking for where
|
|
16
|
+
something is implemented.
|
|
17
|
+
|
|
18
|
+
| page | covers | verified |
|
|
19
|
+
|---|---|---|
|
|
20
|
+
| [Sync model and guards](../../../docs/architecture/sync-model-and-guards.md) | how pull, push, status and switch decide what to write, which guard refuses what, and where in looker_sync.py each lives | 2026-10-09 |
|
|
21
|
+
|
|
22
|
+
<!-- END GENERATED INDEX -->
|
|
23
|
+
|
|
24
|
+
The same table, browsable, is [docs/README.md](../../../docs/README.md). To add a page,
|
|
25
|
+
follow the `repo-maintenance` skill, then run `scripts/gen-docs-index.sh`.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: repo-maintenance
|
|
3
|
+
description: The traps, reference pages and decision records of looker-sync, indexed by symptom. Use before debugging behaviour that is wrong but not crashing, before changing the fake Looker in the tests, and whenever a debugging session cost more than an hour.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Traps and reference
|
|
7
|
+
|
|
8
|
+
Scan the tables below by the symptom you see, then open exactly one page. Each page
|
|
9
|
+
is self-contained.
|
|
10
|
+
|
|
11
|
+
<!-- BEGIN GENERATED INDEX -- edit the pages, not this block -->
|
|
12
|
+
|
|
13
|
+
## Traps
|
|
14
|
+
|
|
15
|
+
Failure modes that produce no error message, indexed by the symptom you
|
|
16
|
+
would observe. Read before debugging behaviour that is wrong but not
|
|
17
|
+
crashing.
|
|
18
|
+
|
|
19
|
+
| symptom | page | area | verified |
|
|
20
|
+
|---|---|---|---|
|
|
21
|
+
| a changed file shows git_status action add, so it looks like a new file | [MODIFIED_FILE_SHOWS_ACTION_ADD](../../../docs/traps/MODIFIED_FILE_SHOWS_ACTION_ADD.md) | file list / git_status | 2026-10-09 |
|
|
22
|
+
| unit tests pass but push fails against real Looker with HTTP 400 No such file or directory | [TESTS_PASS_BUT_PUSH_FAILS_ON_THE_REAL_LOOKER](../../../docs/traps/TESTS_PASS_BUT_PUSH_FAILS_ON_THE_REAL_LOOKER.md) | tests / fake Looker | 2026-10-09 |
|
|
23
|
+
|
|
24
|
+
## Reference
|
|
25
|
+
|
|
26
|
+
Simply true, and expensive to re-derive.
|
|
27
|
+
|
|
28
|
+
| page | summary | verified |
|
|
29
|
+
|---|---|---|
|
|
30
|
+
| [Looker file and branch API behaviour (live-verified)](../../../docs/reference/looker-file-api-behaviour.md) | How the Looker API answers for new folders, directories and the shared dev checkout, as seen on a live instance | 2026-10-09 |
|
|
31
|
+
|
|
32
|
+
## Decisions
|
|
33
|
+
|
|
34
|
+
Why the repo is the way it is. A merged decision is immutable -- supersede
|
|
35
|
+
it with a new one rather than editing it.
|
|
36
|
+
|
|
37
|
+
- [The tool writes files and never touches git state](../../../docs/adr/0001-read-and-write-files-only.md)
|
|
38
|
+
- [No commit command; ship on PyPI with the skill inside](../../../docs/adr/0002-no-commit-command-and-uvx-distribution.md)
|
|
39
|
+
|
|
40
|
+
<!-- END GENERATED INDEX -->
|
|
41
|
+
|
|
42
|
+
## Where a lesson goes
|
|
43
|
+
|
|
44
|
+
Type error, then test, then a comment at the site, then a doc. A doc is the last resort.
|
|
45
|
+
A fact with a symptom is a trap in `docs/traps/`; a fact that is simply true is a page in
|
|
46
|
+
`docs/reference/`; a hard-to-reverse choice is a record in `docs/adr/`. Verify against
|
|
47
|
+
running code, date the page `verified:`, then run `scripts/gen-docs-index.sh`.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 luutuankiet
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: looker-sync
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Pull and push LookML between a local folder and your own Looker dev workspace, with read-back verification.
|
|
5
|
+
Project-URL: Homepage, https://github.com/luutuankiet/looker-sync
|
|
6
|
+
Project-URL: Issues, https://github.com/luutuankiet/looker-sync/issues
|
|
7
|
+
Author: luutuankiet
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: cli,looker,lookml,sync
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# looker-sync
|
|
18
|
+
|
|
19
|
+
Move LookML between a local folder and your own Looker dev workspace, with verification.
|
|
20
|
+
Python 3 standard library only.
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
uvx looker-sync init --project <name> [--branch <b>] # works in a blank folder
|
|
24
|
+
uvx looker-sync status
|
|
25
|
+
uvx looker-sync push --dry-run
|
|
26
|
+
uvx looker-sync push
|
|
27
|
+
uvx looker-sync skill # the full guide, written for agents
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`init` in a folder with no credentials writes an `.env` template and a `.gitignore` entry,
|
|
31
|
+
then asks you to fill it in; run it again to pull.
|
|
32
|
+
|
|
33
|
+
The env file holds `LOOKER_URL`, `LOOKER_CLIENT_ID` and `LOOKER_CLIENT_SECRET`, for an API
|
|
34
|
+
key that belongs to **your own** Looker user.
|
|
35
|
+
|
|
36
|
+
## What it guarantees
|
|
37
|
+
|
|
38
|
+
- `push` writes your changes, reads every file back and compares bytes, then runs LookML
|
|
39
|
+
validation and prints `file:line message`.
|
|
40
|
+
- It refuses to overwrite a file edited in Looker since your last sync (`--force` to override).
|
|
41
|
+
- `pull` never deletes local files and refuses to overwrite unpushed local work.
|
|
42
|
+
- It never commits, deploys, resets or creates branches. Looker's API has no commit or push
|
|
43
|
+
endpoint, so committing stays with git and the Looker IDE.
|
|
44
|
+
- A push that stops partway records what it wrote; the next push retries only the rest.
|
|
45
|
+
|
|
46
|
+
## One key, one branch at a time
|
|
47
|
+
|
|
48
|
+
The checked-out dev branch belongs to the Looker user, not to an API token. A second client on
|
|
49
|
+
another branch is refused by a branch-mismatch guard; only `switch` moves the branch, and the
|
|
50
|
+
browser IDE follows.
|
|
51
|
+
|
|
52
|
+
Exit codes: 0 ok, 1 Looker or network error, 2 refused by a guard, 3 validation errors.
|
|
53
|
+
|
|
54
|
+
Tests: `python3 -m unittest discover -s scripts -p 'test_looker_sync.py'`
|
|
55
|
+
|
|
56
|
+
## Claude plugin
|
|
57
|
+
|
|
58
|
+
The repo is also a Claude plugin: `skills/looker-sync/SKILL.md` is the one authoritative guide,
|
|
59
|
+
shipped in the plugin and inside the package (`looker-sync skill` prints it).
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# looker-sync
|
|
2
|
+
|
|
3
|
+
Move LookML between a local folder and your own Looker dev workspace, with verification.
|
|
4
|
+
Python 3 standard library only.
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
uvx looker-sync init --project <name> [--branch <b>] # works in a blank folder
|
|
8
|
+
uvx looker-sync status
|
|
9
|
+
uvx looker-sync push --dry-run
|
|
10
|
+
uvx looker-sync push
|
|
11
|
+
uvx looker-sync skill # the full guide, written for agents
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
`init` in a folder with no credentials writes an `.env` template and a `.gitignore` entry,
|
|
15
|
+
then asks you to fill it in; run it again to pull.
|
|
16
|
+
|
|
17
|
+
The env file holds `LOOKER_URL`, `LOOKER_CLIENT_ID` and `LOOKER_CLIENT_SECRET`, for an API
|
|
18
|
+
key that belongs to **your own** Looker user.
|
|
19
|
+
|
|
20
|
+
## What it guarantees
|
|
21
|
+
|
|
22
|
+
- `push` writes your changes, reads every file back and compares bytes, then runs LookML
|
|
23
|
+
validation and prints `file:line message`.
|
|
24
|
+
- It refuses to overwrite a file edited in Looker since your last sync (`--force` to override).
|
|
25
|
+
- `pull` never deletes local files and refuses to overwrite unpushed local work.
|
|
26
|
+
- It never commits, deploys, resets or creates branches. Looker's API has no commit or push
|
|
27
|
+
endpoint, so committing stays with git and the Looker IDE.
|
|
28
|
+
- A push that stops partway records what it wrote; the next push retries only the rest.
|
|
29
|
+
|
|
30
|
+
## One key, one branch at a time
|
|
31
|
+
|
|
32
|
+
The checked-out dev branch belongs to the Looker user, not to an API token. A second client on
|
|
33
|
+
another branch is refused by a branch-mismatch guard; only `switch` moves the branch, and the
|
|
34
|
+
browser IDE follows.
|
|
35
|
+
|
|
36
|
+
Exit codes: 0 ok, 1 Looker or network error, 2 refused by a guard, 3 validation errors.
|
|
37
|
+
|
|
38
|
+
Tests: `python3 -m unittest discover -s scripts -p 'test_looker_sync.py'`
|
|
39
|
+
|
|
40
|
+
## Claude plugin
|
|
41
|
+
|
|
42
|
+
The repo is also a Claude plugin: `skills/looker-sync/SKILL.md` is the one authoritative guide,
|
|
43
|
+
shipped in the plugin and inside the package (`looker-sync skill` prints it).
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Documentation
|
|
2
|
+
|
|
3
|
+
Every page here is written for a maintainer six months from now who opened
|
|
4
|
+
exactly this file from a search result and has nothing else loaded.
|
|
5
|
+
|
|
6
|
+
This index is generated. Run `scripts/gen-docs-index.sh` after adding or
|
|
7
|
+
renaming a page; `--check` fails if it is stale.
|
|
8
|
+
|
|
9
|
+
<!-- BEGIN GENERATED INDEX -- edit the pages, not this block -->
|
|
10
|
+
|
|
11
|
+
## Where things live
|
|
12
|
+
|
|
13
|
+
One page per area of the system. Read before going looking for where
|
|
14
|
+
something is implemented.
|
|
15
|
+
|
|
16
|
+
| page | covers | verified |
|
|
17
|
+
|---|---|---|
|
|
18
|
+
| [Sync model and guards](architecture/sync-model-and-guards.md) | how pull, push, status and switch decide what to write, which guard refuses what, and where in looker_sync.py each lives | 2026-10-09 |
|
|
19
|
+
|
|
20
|
+
## Traps
|
|
21
|
+
|
|
22
|
+
Failure modes that produce no error message, indexed by the symptom you
|
|
23
|
+
would observe. Read before debugging behaviour that is wrong but not
|
|
24
|
+
crashing.
|
|
25
|
+
|
|
26
|
+
| symptom | page | area | verified |
|
|
27
|
+
|---|---|---|---|
|
|
28
|
+
| a changed file shows git_status action add, so it looks like a new file | [MODIFIED_FILE_SHOWS_ACTION_ADD](traps/MODIFIED_FILE_SHOWS_ACTION_ADD.md) | file list / git_status | 2026-10-09 |
|
|
29
|
+
| unit tests pass but push fails against real Looker with HTTP 400 No such file or directory | [TESTS_PASS_BUT_PUSH_FAILS_ON_THE_REAL_LOOKER](traps/TESTS_PASS_BUT_PUSH_FAILS_ON_THE_REAL_LOOKER.md) | tests / fake Looker | 2026-10-09 |
|
|
30
|
+
|
|
31
|
+
## Reference
|
|
32
|
+
|
|
33
|
+
Simply true, and expensive to re-derive.
|
|
34
|
+
|
|
35
|
+
| page | summary | verified |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| [Looker file and branch API behaviour (live-verified)](reference/looker-file-api-behaviour.md) | How the Looker API answers for new folders, directories and the shared dev checkout, as seen on a live instance | 2026-10-09 |
|
|
38
|
+
|
|
39
|
+
## Decisions
|
|
40
|
+
|
|
41
|
+
Why the repo is the way it is. A merged decision is immutable -- supersede
|
|
42
|
+
it with a new one rather than editing it.
|
|
43
|
+
|
|
44
|
+
- [The tool writes files and never touches git state](adr/0001-read-and-write-files-only.md)
|
|
45
|
+
- [No commit command; ship on PyPI with the skill inside](adr/0002-no-commit-command-and-uvx-distribution.md)
|
|
46
|
+
|
|
47
|
+
<!-- END GENERATED INDEX -->
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "looker-sync"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Pull and push LookML between a local folder and your own Looker dev workspace, with read-back verification."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
authors = [{ name = "luutuankiet" }]
|
|
14
|
+
keywords = ["looker", "lookml", "sync", "cli"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
]
|
|
20
|
+
dependencies = []
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://github.com/luutuankiet/looker-sync"
|
|
24
|
+
Issues = "https://github.com/luutuankiet/looker-sync/issues"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
looker-sync = "looker_sync.cli:main"
|
|
28
|
+
|
|
29
|
+
[tool.hatch.build.targets.wheel]
|
|
30
|
+
packages = ["src/looker_sync"]
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.wheel.force-include]
|
|
33
|
+
"skills/looker-sync" = "looker_sync/_skills_data/looker-sync"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.targets.sdist]
|
|
36
|
+
include = ["src", "skills", "README.md", "LICENSE", "pyproject.toml"]
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: looker-sync
|
|
3
|
+
description: Move LookML between a local folder and your own Looker dev workspace with the looker-sync CLI (pull, status, push with byte-for-byte read-back and validation, switch). Use before editing any LookML that must reach Looker, before validating a Looker project, when mounting a Looker project into a new folder, and when Looker and the local files might disagree.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# looker-sync
|
|
7
|
+
|
|
8
|
+
`looker-sync` is the only path from local LookML to Looker. It talks to the Looker API with
|
|
9
|
+
your own key. It never commits, deploys, resets or creates branches: Looker's API has no
|
|
10
|
+
commit or push endpoint, so committing stays with git, and the rest stay clicks in the Looker
|
|
11
|
+
IDE. Agents cannot commit LookML through this tool.
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
uvx looker-sync [-C <project folder>] [--env-file <path>] <command>
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
`-C` defaults to the current folder. The env file holds `LOOKER_URL`, `LOOKER_CLIENT_ID`
|
|
18
|
+
and `LOOKER_CLIENT_SECRET`. It is found as `--env-file`, then `LOOKER_SYNC_ENV`, then the
|
|
19
|
+
`env_file` in `.looker-sync/config.json`, then `.env` in the project folder. Never open it
|
|
20
|
+
yourself. `looker-sync --help` is the short reference; `looker-sync skill` prints this guide.
|
|
21
|
+
|
|
22
|
+
## The loop
|
|
23
|
+
|
|
24
|
+
1. `status`: what differs. Nothing is written.
|
|
25
|
+
2. Edit the files with your normal tools.
|
|
26
|
+
3. `push --dry-run`, then `push`. Push writes local changes, reads every file back, compares
|
|
27
|
+
bytes, then runs LookML validation and prints errors as `file:line message`.
|
|
28
|
+
4. Fix, push again, until exit 0.
|
|
29
|
+
|
|
30
|
+
| exit | meaning | what to do |
|
|
31
|
+
|---|---|---|
|
|
32
|
+
| 0 | ok | carry on |
|
|
33
|
+
| 1 | Looker or network error, or a read-back mismatch | read the message; do not retry blindly |
|
|
34
|
+
| 2 | refused by a guard | read the reason; it names the files |
|
|
35
|
+
| 3 | pushed and verified, but validation has errors | fix them; check whether they were there before your push |
|
|
36
|
+
|
|
37
|
+
**Read the first line of every run.** It names the Looker user, project, mode and branch.
|
|
38
|
+
If the user is not the developer you are working for, stop: a key for another user writes
|
|
39
|
+
to that user's dev workspace, which the developer never sees.
|
|
40
|
+
|
|
41
|
+
## Guards, and what each refusal means
|
|
42
|
+
|
|
43
|
+
- **Branch mismatch.** Looker's dev workspace is on another branch than the folder's config.
|
|
44
|
+
Someone switched in the IDE. Stop and tell the user; only `switch` moves branches.
|
|
45
|
+
- **Changed in Looker since the last sync** (push). Someone edited in the Looker IDE. Run
|
|
46
|
+
`pull` to take their copy if you have not touched that file; otherwise report it. Use
|
|
47
|
+
`push --force` only when the user has said local wins.
|
|
48
|
+
- **Pull would overwrite unpushed local changes.** Push your work first. `pull --force`
|
|
49
|
+
throws it away.
|
|
50
|
+
- **Push refused in prod mode.** Production is read-only by design.
|
|
51
|
+
- **One key, one branch at a time.** The checked-out branch belongs to the Looker user, not
|
|
52
|
+
to a token: a second client or folder on another branch hits the branch-mismatch guard, and
|
|
53
|
+
only `switch` moves everyone. Do not run two branches in parallel on one key.
|
|
54
|
+
|
|
55
|
+
## Rules
|
|
56
|
+
|
|
57
|
+
- A new folder is created for you. A push writes new files before changed ones and records
|
|
58
|
+
each file as soon as it is written and read back. If it stops partway it prints `written:`
|
|
59
|
+
and `not written:`; just push again, no `--force` is needed.
|
|
60
|
+
- `--force` and `--delete` need a stated reason in your reply. Files that exist only in
|
|
61
|
+
Looker are listed by every push and deleted only with `--delete`.
|
|
62
|
+
- `pull` never deletes local files, and keeps a file you deleted locally deleted.
|
|
63
|
+
- `switch` moves the user's whole dev workspace, browser IDE included. Leave it to the user
|
|
64
|
+
unless they asked. It refuses while any Looker file is uncommitted, and without a terminal
|
|
65
|
+
it needs `--yes` plus `--local-changes push|backup|cancel` when local work is unpushed.
|
|
66
|
+
Backups land in `.looker-sync/backups/<branch>-<timestamp>/`.
|
|
67
|
+
- Only `*.lkml` and `*.dashboard.lookml` sync by default (`include`/`exclude` in the
|
|
68
|
+
config). `.env`, skills, editor settings and `.looker-sync/` never reach Looker.
|
|
69
|
+
- The Looker MCP write tools are not an alternative: they run in production mode under a
|
|
70
|
+
separate login and cannot see dev files. MCP read and query tools are fine.
|
|
71
|
+
|
|
72
|
+
## Mounting a project, including from a blank folder
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
mkdir my-project && cd my-project
|
|
76
|
+
uvx looker-sync init --project <name> [--branch <b>]
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
In a folder with no credentials, `init` writes a `.env` template (owner-only permissions),
|
|
80
|
+
adds `.env` and `.looker-sync/` to `.gitignore`, and exits with code 2 telling you what to
|
|
81
|
+
fill in. The user fills `LOOKER_URL`, `LOOKER_CLIENT_ID` and `LOOKER_CLIENT_SECRET`; run the
|
|
82
|
+
same `init` command again and it writes `.looker-sync/config.json` and pulls. It never
|
|
83
|
+
overwrites a filled `.env` or an existing config. On a terminal it asks for the project and
|
|
84
|
+
whether to pull; `--no-pull` writes metadata only.
|
|
85
|
+
|
|
86
|
+
`init` refuses if the Looker dev workspace is on a different branch than `--branch` (it
|
|
87
|
+
never switches), and its first pull refuses to overwrite local files that differ from
|
|
88
|
+
Looker (`--force` takes Looker's copy). Mount into an empty folder and compare by hand when
|
|
89
|
+
in doubt: a folder kept by another sync tool can be stale.
|
|
90
|
+
|
|
91
|
+
## Committing
|
|
92
|
+
|
|
93
|
+
`looker-sync` does not commit. `status` prints how many commits the dev branch is ahead of
|
|
94
|
+
and behind its remote; commit and push with git from the Looker IDE or a clone of the
|
|
95
|
+
remote. Looker has no commit or push endpoint in its API, and the official Looker VS Code
|
|
96
|
+
extension (1.0.0) does not use one either: it commits through local git.
|
|
97
|
+
|
|
98
|
+
## Tests
|
|
99
|
+
|
|
100
|
+
From a source checkout, `python3 -m unittest discover -s scripts -p 'test_looker_sync.py'`
|
|
101
|
+
runs the CLI against a fake Looker. Run it after any change to the tool.
|
|
File without changes
|
|
@@ -0,0 +1,740 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""looker-sync: pull and push LookML between a local folder and a Looker dev workspace.
|
|
3
|
+
|
|
4
|
+
Talks to the Looker REST API (4.0) directly with the user's own API key. Standard library
|
|
5
|
+
only. Never commits, deploys, resets or creates branches: Looker's API has no commit or push
|
|
6
|
+
endpoint, so those stay with git and the Looker IDE.
|
|
7
|
+
|
|
8
|
+
looker-sync [-C PROJECT_DIR] [--env-file PATH] COMMAND ...
|
|
9
|
+
|
|
10
|
+
init [--project P] [--mode dev|prod] [--branch B] set up a folder, then a first pull
|
|
11
|
+
pull [--force] [--dry-run] copy Looker files to disk
|
|
12
|
+
status show what differs on each side
|
|
13
|
+
push [--force] [--delete] [--dry-run] write local edits, read back, validate
|
|
14
|
+
switch (--branch B | --mode dev|prod) [--local-changes push|backup|cancel] [--yes]
|
|
15
|
+
skill [--path] print the full usage guide for agents
|
|
16
|
+
|
|
17
|
+
Metadata lives in PROJECT_DIR/.looker-sync/ (config.json, state.json, backups/). Credentials
|
|
18
|
+
come only from the env file: LOOKER_URL, LOOKER_CLIENT_ID, LOOKER_CLIENT_SECRET.
|
|
19
|
+
|
|
20
|
+
Exit codes: 0 ok, 1 Looker or network error, 2 refused by a guard, 3 validation errors.
|
|
21
|
+
"""
|
|
22
|
+
import argparse
|
|
23
|
+
import datetime
|
|
24
|
+
import hashlib
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import shutil
|
|
29
|
+
import sys
|
|
30
|
+
import urllib.error
|
|
31
|
+
import urllib.parse
|
|
32
|
+
import urllib.request
|
|
33
|
+
|
|
34
|
+
META_DIR = '.looker-sync'
|
|
35
|
+
DEFAULT_INCLUDE = ['**/*.lkml', '**/*.dashboard.lookml']
|
|
36
|
+
OK, LOOKER_ERROR, REFUSED, INVALID = 0, 1, 2, 3
|
|
37
|
+
WORKSPACE = {'dev': 'dev', 'prod': 'production'}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class LookerError(Exception):
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Refused(Exception):
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def out(msg=''):
|
|
49
|
+
print(msg, flush=True)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def sha(data):
|
|
53
|
+
return None if data is None else hashlib.sha256(data).hexdigest()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------- env and API
|
|
57
|
+
|
|
58
|
+
def load_env(path):
|
|
59
|
+
if not os.path.isfile(path):
|
|
60
|
+
raise LookerError(f'env file not found: {path}')
|
|
61
|
+
env = {}
|
|
62
|
+
with open(path) as f:
|
|
63
|
+
for line in f:
|
|
64
|
+
line = line.strip()
|
|
65
|
+
if line.startswith('export '):
|
|
66
|
+
line = line[len('export '):]
|
|
67
|
+
if not line or line.startswith('#') or '=' not in line:
|
|
68
|
+
continue
|
|
69
|
+
k, v = line.split('=', 1)
|
|
70
|
+
env[k.strip()] = v.strip().strip('"').strip("'")
|
|
71
|
+
missing = [k for k in ('LOOKER_URL', 'LOOKER_CLIENT_ID', 'LOOKER_CLIENT_SECRET') if not env.get(k)]
|
|
72
|
+
if missing:
|
|
73
|
+
raise LookerError(f'env file {path} lacks {", ".join(missing)}')
|
|
74
|
+
return env
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Looker:
|
|
78
|
+
"""One login per command run. The token lives in memory only."""
|
|
79
|
+
|
|
80
|
+
def __init__(self, env):
|
|
81
|
+
url = env['LOOKER_URL'].rstrip('/')
|
|
82
|
+
self.base = url if url.endswith('/api/4.0') else url + '/api/4.0'
|
|
83
|
+
body = urllib.parse.urlencode({'client_id': env['LOOKER_CLIENT_ID'],
|
|
84
|
+
'client_secret': env['LOOKER_CLIENT_SECRET']}).encode()
|
|
85
|
+
status, payload = self._send('POST', '/login', body, {'Content-Type': 'application/x-www-form-urlencoded'})
|
|
86
|
+
if status != 200:
|
|
87
|
+
# The request carried the secret; the response never does, but keep it terse anyway.
|
|
88
|
+
raise LookerError(f'login failed: HTTP {status}')
|
|
89
|
+
self.token = json.loads(payload)['access_token']
|
|
90
|
+
|
|
91
|
+
def _send(self, method, path, data=None, headers=None):
|
|
92
|
+
req = urllib.request.Request(self.base + path, data=data, method=method, headers=headers or {})
|
|
93
|
+
try:
|
|
94
|
+
with urllib.request.urlopen(req, timeout=120) as r:
|
|
95
|
+
return r.status, r.read()
|
|
96
|
+
except urllib.error.HTTPError as e:
|
|
97
|
+
return e.code, e.read()
|
|
98
|
+
except (urllib.error.URLError, OSError) as e:
|
|
99
|
+
raise LookerError(f'{method} {path}: {e}')
|
|
100
|
+
|
|
101
|
+
def try_call(self, method, path, query=None, body=None, raw=False):
|
|
102
|
+
if query:
|
|
103
|
+
path += '?' + urllib.parse.urlencode(query)
|
|
104
|
+
headers = {'Authorization': 'token ' + self.token}
|
|
105
|
+
data = None
|
|
106
|
+
if body is not None:
|
|
107
|
+
data = json.dumps(body).encode()
|
|
108
|
+
headers['Content-Type'] = 'application/json'
|
|
109
|
+
status, payload = self._send(method, path, data, headers)
|
|
110
|
+
if status >= 400:
|
|
111
|
+
try:
|
|
112
|
+
msg = json.loads(payload).get('message') or payload.decode(errors='replace')
|
|
113
|
+
except (ValueError, AttributeError):
|
|
114
|
+
msg = payload.decode(errors='replace')
|
|
115
|
+
return status, msg[:300]
|
|
116
|
+
if raw:
|
|
117
|
+
return status, payload
|
|
118
|
+
return status, json.loads(payload) if payload else None
|
|
119
|
+
|
|
120
|
+
def call(self, method, path, query=None, body=None, raw=False):
|
|
121
|
+
status, payload = self.try_call(method, path, query, body, raw)
|
|
122
|
+
if status >= 400:
|
|
123
|
+
raise LookerError(f'{method} {path.split("?")[0]} -> HTTP {status}: {payload}')
|
|
124
|
+
return payload
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------------------------------------------------------------- project folder
|
|
128
|
+
|
|
129
|
+
def glob_regex(pattern):
|
|
130
|
+
i, rx = 0, ''
|
|
131
|
+
while i < len(pattern):
|
|
132
|
+
if pattern.startswith('**/', i):
|
|
133
|
+
rx, i = rx + '(?:.*/)?', i + 3
|
|
134
|
+
elif pattern.startswith('**', i):
|
|
135
|
+
rx, i = rx + '.*', i + 2
|
|
136
|
+
elif pattern[i] == '*':
|
|
137
|
+
rx, i = rx + '[^/]*', i + 1
|
|
138
|
+
elif pattern[i] == '?':
|
|
139
|
+
rx, i = rx + '[^/]', i + 1
|
|
140
|
+
else:
|
|
141
|
+
rx, i = rx + re.escape(pattern[i]), i + 1
|
|
142
|
+
return re.compile(rx + r'\Z')
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class Project:
|
|
146
|
+
def __init__(self, root):
|
|
147
|
+
self.root = os.path.abspath(root)
|
|
148
|
+
self.meta = os.path.join(self.root, META_DIR)
|
|
149
|
+
self.config_path = os.path.join(self.meta, 'config.json')
|
|
150
|
+
self.state_path = os.path.join(self.meta, 'state.json')
|
|
151
|
+
self.config = self._read(self.config_path)
|
|
152
|
+
self.state = self._read(self.state_path) or {'files': {}}
|
|
153
|
+
|
|
154
|
+
@staticmethod
|
|
155
|
+
def _read(path):
|
|
156
|
+
if not os.path.isfile(path):
|
|
157
|
+
return None
|
|
158
|
+
with open(path) as f:
|
|
159
|
+
return json.load(f)
|
|
160
|
+
|
|
161
|
+
def _write(self, path, data):
|
|
162
|
+
os.makedirs(self.meta, exist_ok=True)
|
|
163
|
+
tmp = path + '.tmp'
|
|
164
|
+
with open(tmp, 'w') as f:
|
|
165
|
+
json.dump(data, f, indent=2, sort_keys=True)
|
|
166
|
+
f.write('\n')
|
|
167
|
+
os.replace(tmp, path)
|
|
168
|
+
|
|
169
|
+
def save_config(self):
|
|
170
|
+
self._write(self.config_path, self.config)
|
|
171
|
+
|
|
172
|
+
def save_state(self):
|
|
173
|
+
self._write(self.state_path, self.state)
|
|
174
|
+
|
|
175
|
+
def require_config(self):
|
|
176
|
+
if not self.config:
|
|
177
|
+
raise Refused(f'no {META_DIR}/config.json in {self.root}; run init first')
|
|
178
|
+
return self.config
|
|
179
|
+
|
|
180
|
+
def synced(self, path):
|
|
181
|
+
"""True if the path is in the sync set. The metadata folder never is."""
|
|
182
|
+
if path.split('/')[0] == META_DIR:
|
|
183
|
+
return False
|
|
184
|
+
cfg = self.config or {}
|
|
185
|
+
inc = [glob_regex(p) for p in cfg.get('include', DEFAULT_INCLUDE)]
|
|
186
|
+
exc = [glob_regex(p) for p in cfg.get('exclude', [])]
|
|
187
|
+
return any(r.match(path) for r in inc) and not any(r.match(path) for r in exc)
|
|
188
|
+
|
|
189
|
+
def local_files(self):
|
|
190
|
+
found = {}
|
|
191
|
+
for dirpath, dirnames, filenames in os.walk(self.root):
|
|
192
|
+
rel_dir = os.path.relpath(dirpath, self.root)
|
|
193
|
+
if rel_dir == '.':
|
|
194
|
+
dirnames[:] = [d for d in dirnames if d != META_DIR]
|
|
195
|
+
for name in filenames:
|
|
196
|
+
rel = name if rel_dir == '.' else f'{rel_dir}/{name}'.replace(os.sep, '/')
|
|
197
|
+
if self.synced(rel):
|
|
198
|
+
with open(os.path.join(dirpath, name), 'rb') as f:
|
|
199
|
+
found[rel] = f.read()
|
|
200
|
+
return found
|
|
201
|
+
|
|
202
|
+
def write_local(self, path, data):
|
|
203
|
+
full = os.path.join(self.root, *path.split('/'))
|
|
204
|
+
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
205
|
+
with open(full, 'wb') as f:
|
|
206
|
+
f.write(data)
|
|
207
|
+
|
|
208
|
+
def env_path(self, flag):
|
|
209
|
+
flag = flag or os.environ.get('LOOKER_SYNC_ENV')
|
|
210
|
+
if flag:
|
|
211
|
+
return flag
|
|
212
|
+
if self.config and self.config.get('env_file'):
|
|
213
|
+
return os.path.join(self.root, self.config['env_file'])
|
|
214
|
+
return os.path.join(self.root, '.env')
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ---------------------------------------------------------------- session
|
|
218
|
+
|
|
219
|
+
class Session:
|
|
220
|
+
"""Log in, set the workspace from the config mode, confirm it, print who and where."""
|
|
221
|
+
|
|
222
|
+
def __init__(self, proj, env_flag, mode, project, branch, check_branch=True):
|
|
223
|
+
self.proj, self.project, self.mode = proj, project, mode
|
|
224
|
+
self.L = Looker(load_env(proj.env_path(env_flag)))
|
|
225
|
+
self.set_mode(mode)
|
|
226
|
+
u = self.L.call('GET', '/user')
|
|
227
|
+
self.user = f'{u.get("display_name")} <{u.get("email")}> [id {u.get("id")}]'
|
|
228
|
+
self.branch = self.current_branch()
|
|
229
|
+
shown = self.branch if mode == 'dev' else f'{self.branch} (production)'
|
|
230
|
+
out(f'looker-sync: as {self.user} · project {project} · mode {mode} · branch {shown}')
|
|
231
|
+
if mode == 'dev' and check_branch and branch != self.branch:
|
|
232
|
+
raise Refused(f'branch mismatch: config says {branch!r}, Looker dev workspace is on '
|
|
233
|
+
f'{self.branch!r}. Run switch, or check out {branch!r} in the Looker IDE.')
|
|
234
|
+
|
|
235
|
+
def set_mode(self, mode):
|
|
236
|
+
want = WORKSPACE[mode]
|
|
237
|
+
self.L.call('PATCH', '/session', body={'workspace_id': want})
|
|
238
|
+
got = (self.L.call('GET', '/session') or {}).get('workspace_id')
|
|
239
|
+
if got != want:
|
|
240
|
+
raise LookerError(f'session workspace is {got!r} after asking for {want!r}')
|
|
241
|
+
self.mode = mode
|
|
242
|
+
|
|
243
|
+
def current_branch(self):
|
|
244
|
+
status, b = self.L.try_call('GET', f'/projects/{self.project}/git_branch')
|
|
245
|
+
if status >= 400:
|
|
246
|
+
if self.mode == 'dev':
|
|
247
|
+
raise LookerError(f'GET git_branch -> HTTP {status}: {b}')
|
|
248
|
+
return None
|
|
249
|
+
self.branch_info = b
|
|
250
|
+
return b.get('name')
|
|
251
|
+
|
|
252
|
+
def file_list(self):
|
|
253
|
+
return self.L.call('GET', f'/projects/{self.project}/files') or []
|
|
254
|
+
|
|
255
|
+
def remote_files(self):
|
|
256
|
+
"""{path: bytes} for every Looker file in the sync set."""
|
|
257
|
+
files = {}
|
|
258
|
+
for f in self.file_list():
|
|
259
|
+
if self.proj.synced(f['path']):
|
|
260
|
+
files[f['path']] = self.read(f['path'])
|
|
261
|
+
return files
|
|
262
|
+
|
|
263
|
+
def read(self, path):
|
|
264
|
+
return self.L.call('GET', f'/projects/{self.project}/file/content', {'file_path': path}, raw=True)
|
|
265
|
+
|
|
266
|
+
def write(self, path, data, exists):
|
|
267
|
+
body = {'path': path, 'content': data.decode('utf-8')}
|
|
268
|
+
base = f'/projects/{self.project}/files'
|
|
269
|
+
if exists:
|
|
270
|
+
return self.L.call('PUT', base, body=body)
|
|
271
|
+
status, msg = self.L.try_call('POST', base, body=body)
|
|
272
|
+
# Looker answers 400 'No such file or directory' for a missing folder; older/other builds 404
|
|
273
|
+
missing_folder = status == 404 or (status == 400 and 'No such file or directory' in str(msg))
|
|
274
|
+
if missing_folder and '/' in path:
|
|
275
|
+
parts = path.split('/')[:-1]
|
|
276
|
+
for i in range(1, len(parts) + 1):
|
|
277
|
+
self.L.try_call('POST', f'/projects/{self.project}/directories', body={'path': '/'.join(parts[:i])})
|
|
278
|
+
status, msg = self.L.try_call('POST', base, body=body)
|
|
279
|
+
if status >= 400:
|
|
280
|
+
raise LookerError(f'POST {base} {path} -> HTTP {status}: {msg}')
|
|
281
|
+
|
|
282
|
+
def delete(self, path):
|
|
283
|
+
self.L.call('DELETE', f'/projects/{self.project}/files', {'file_path': path})
|
|
284
|
+
|
|
285
|
+
def validate(self):
|
|
286
|
+
res = self.L.call('POST', f'/projects/{self.project}/validate') or {}
|
|
287
|
+
errors = 0
|
|
288
|
+
for e in res.get('errors') or []:
|
|
289
|
+
sev = e.get('severity') or 'error'
|
|
290
|
+
where = e.get('source_file') or e.get('file_path') or f'(no file; model {e.get("model_id") or "?"})'
|
|
291
|
+
if e.get('line_number') or e.get('line'):
|
|
292
|
+
where += f':{e.get("line_number") or e.get("line")}'
|
|
293
|
+
out(f' {sev:7} {where} {e.get("message")}')
|
|
294
|
+
if sev in ('error', 'fatal'):
|
|
295
|
+
errors += 1
|
|
296
|
+
out(f'validation: {errors} error(s)' if errors else 'validation: no errors')
|
|
297
|
+
return errors
|
|
298
|
+
|
|
299
|
+
def uncommitted(self):
|
|
300
|
+
return [(f['path'], f['git_status'].get('text') or f['git_status'].get('action'))
|
|
301
|
+
for f in self.file_list() if f.get('git_status')]
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def open_session(proj, args, check_branch=True):
|
|
305
|
+
cfg = proj.require_config()
|
|
306
|
+
return Session(proj, args.env_file, cfg['mode'], cfg['project'], cfg.get('branch'), check_branch)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# ---------------------------------------------------------------- sync plan
|
|
310
|
+
|
|
311
|
+
KIND_TEXT = {
|
|
312
|
+
'local-new': 'only local (push creates it)',
|
|
313
|
+
'local-modified': 'changed locally (push writes it)',
|
|
314
|
+
'local-deleted': 'deleted locally (push --delete removes it from Looker)',
|
|
315
|
+
'remote-new': 'only in Looker (pull writes it; push --delete removes it)',
|
|
316
|
+
'remote-modified': 'changed in Looker since last sync (pull writes it)',
|
|
317
|
+
'remote-deleted': 'deleted in Looker since last sync (pull keeps the local copy)',
|
|
318
|
+
'conflict': 'changed on both sides',
|
|
319
|
+
}
|
|
320
|
+
REMOTE_CHANGED = ('remote-modified', 'remote-deleted', 'conflict')
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def plan(local, remote, known):
|
|
324
|
+
"""{path: kind} for every path that differs. known = {path: sha at last sync}."""
|
|
325
|
+
kinds = {}
|
|
326
|
+
for path in sorted(set(local) | set(remote) | set(known)):
|
|
327
|
+
lh, rh, s = sha(local.get(path)), sha(remote.get(path)), known.get(path)
|
|
328
|
+
if lh == rh:
|
|
329
|
+
continue
|
|
330
|
+
lc, rc = lh != s, rh != s
|
|
331
|
+
if s is None:
|
|
332
|
+
kinds[path] = 'remote-new' if lh is None else 'local-new' if rh is None else 'conflict'
|
|
333
|
+
elif lc and rc:
|
|
334
|
+
kinds[path] = 'conflict'
|
|
335
|
+
elif lc:
|
|
336
|
+
kinds[path] = 'local-deleted' if lh is None else 'local-modified'
|
|
337
|
+
else:
|
|
338
|
+
kinds[path] = 'remote-deleted' if rh is None else 'remote-modified'
|
|
339
|
+
return kinds
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def show(kinds, header):
|
|
343
|
+
if not kinds:
|
|
344
|
+
out(f'{header}: nothing differs')
|
|
345
|
+
return
|
|
346
|
+
out(f'{header}:')
|
|
347
|
+
for kind in KIND_TEXT:
|
|
348
|
+
paths = [p for p, k in kinds.items() if k == kind]
|
|
349
|
+
if paths:
|
|
350
|
+
out(f' {KIND_TEXT[kind]}:')
|
|
351
|
+
for p in paths:
|
|
352
|
+
out(f' {p}')
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def record(proj, sess, updates, drops=()):
|
|
356
|
+
files = proj.state.setdefault('files', {})
|
|
357
|
+
files.update(updates)
|
|
358
|
+
for p in drops:
|
|
359
|
+
files.pop(p, None)
|
|
360
|
+
proj.state.update({'mode': sess.mode, 'branch': sess.branch if sess.mode == 'dev' else None,
|
|
361
|
+
'production_branch': sess.branch if sess.mode == 'prod' else None,
|
|
362
|
+
'user': sess.user,
|
|
363
|
+
'synced_at': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')})
|
|
364
|
+
proj.save_state()
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def in_sync(local, remote):
|
|
368
|
+
return {p: sha(d) for p, d in remote.items() if local.get(p) == d}
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
# ---------------------------------------------------------------- commands
|
|
372
|
+
|
|
373
|
+
def do_pull(proj, sess, force=False, dry_run=False):
|
|
374
|
+
local, remote = proj.local_files(), sess.remote_files()
|
|
375
|
+
kinds = plan(local, remote, proj.state.get('files', {}))
|
|
376
|
+
blocked = [p for p, k in kinds.items() if k == 'conflict' and p in remote]
|
|
377
|
+
writes = [p for p, k in kinds.items() if k in ('remote-new', 'remote-modified')]
|
|
378
|
+
show(kinds, 'pull plan')
|
|
379
|
+
if blocked and not force:
|
|
380
|
+
raise Refused('pull would overwrite unpushed local changes in: ' + ', '.join(blocked)
|
|
381
|
+
+ '. Push them, back them up, or pull --force to take Looker\'s copy.')
|
|
382
|
+
writes += blocked
|
|
383
|
+
if dry_run:
|
|
384
|
+
out(f'dry run: would write {len(writes)} file(s); nothing written')
|
|
385
|
+
return OK
|
|
386
|
+
for p in writes:
|
|
387
|
+
proj.write_local(p, remote[p])
|
|
388
|
+
out(f' wrote {p}')
|
|
389
|
+
updates = in_sync(local, remote)
|
|
390
|
+
updates.update({p: sha(remote[p]) for p in writes})
|
|
391
|
+
record(proj, sess, updates)
|
|
392
|
+
out(f'pull: wrote {len(writes)} file(s)')
|
|
393
|
+
return OK
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
ENV_TEMPLATE = """# Looker API credentials for looker-sync. Never commit this file.
|
|
397
|
+
# Create an API key in Looker: Admin > Users > your user > Edit Keys.
|
|
398
|
+
LOOKER_URL=https://<your-instance>.looker.com
|
|
399
|
+
LOOKER_CLIENT_ID=
|
|
400
|
+
LOOKER_CLIENT_SECRET=
|
|
401
|
+
"""
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def interactive():
|
|
405
|
+
return sys.stdin.isatty() and sys.stdout.isatty()
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def ask(prompt, default=None):
|
|
409
|
+
shown = f'{prompt} [{default}]: ' if default else f'{prompt}: '
|
|
410
|
+
return input(shown).strip() or default
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def ensure_gitignore(root, lines):
|
|
414
|
+
path = os.path.join(root, '.gitignore')
|
|
415
|
+
have = open(path).read().splitlines() if os.path.isfile(path) else []
|
|
416
|
+
add = [l for l in lines if l not in have]
|
|
417
|
+
if add:
|
|
418
|
+
with open(path, 'a') as f:
|
|
419
|
+
if have and not open(path).read().endswith('\n'):
|
|
420
|
+
f.write('\n')
|
|
421
|
+
f.write('\n'.join(add) + '\n')
|
|
422
|
+
out(f'added {", ".join(add)} to .gitignore')
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def env_ready(path):
|
|
426
|
+
"""True when the env file exists and holds real values, not the template blanks."""
|
|
427
|
+
if not os.path.isfile(path):
|
|
428
|
+
return False
|
|
429
|
+
try:
|
|
430
|
+
load_env(path)
|
|
431
|
+
except LookerError:
|
|
432
|
+
return False
|
|
433
|
+
with open(path) as f:
|
|
434
|
+
return '<your-instance>' not in f.read()
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def cmd_init(proj, args):
|
|
438
|
+
resume = bool(proj.config) and not proj.state.get('synced_at')
|
|
439
|
+
if proj.config and not resume:
|
|
440
|
+
raise Refused(f'{proj.config_path} already exists; edit it or remove it first')
|
|
441
|
+
project = (proj.config or {}).get('project') or args.project
|
|
442
|
+
if not project and interactive():
|
|
443
|
+
project = ask('Looker project id (the name in the Looker IDE)')
|
|
444
|
+
if not project:
|
|
445
|
+
raise Refused('init needs --project <looker project id>')
|
|
446
|
+
env_path = proj.env_path(args.env_file)
|
|
447
|
+
if not env_ready(env_path):
|
|
448
|
+
if not os.path.isfile(env_path):
|
|
449
|
+
os.makedirs(os.path.dirname(os.path.abspath(env_path)), exist_ok=True)
|
|
450
|
+
fd = os.open(env_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
451
|
+
with os.fdopen(fd, 'w') as f:
|
|
452
|
+
f.write(ENV_TEMPLATE)
|
|
453
|
+
out(f'wrote {env_path} (a template; owner-only permissions)')
|
|
454
|
+
if os.path.abspath(env_path).startswith(proj.root + os.sep):
|
|
455
|
+
ensure_gitignore(proj.root, [os.path.relpath(env_path, proj.root), META_DIR + '/'])
|
|
456
|
+
raise Refused(f'fill in LOOKER_URL, LOOKER_CLIENT_ID and LOOKER_CLIENT_SECRET in {env_path}, '
|
|
457
|
+
f'then run: looker-sync init --project {project}')
|
|
458
|
+
mode = (proj.config or {}).get('mode') or args.mode
|
|
459
|
+
sess = Session(proj, args.env_file, mode, project, args.branch, check_branch=False)
|
|
460
|
+
branch = (proj.config or {}).get('branch') or args.branch or sess.branch
|
|
461
|
+
if mode == 'dev' and branch != sess.branch:
|
|
462
|
+
raise Refused(f'Looker dev workspace is on {sess.branch!r}, not {branch!r}. init never '
|
|
463
|
+
'switches; check out the branch in the Looker IDE first, or omit --branch.')
|
|
464
|
+
if not proj.config:
|
|
465
|
+
env_flag = args.env_file
|
|
466
|
+
env_rel = os.path.relpath(os.path.abspath(env_flag), proj.root) if env_flag else '.env'
|
|
467
|
+
proj.config = {'project': project, 'mode': mode, 'branch': branch if mode == 'dev' else None,
|
|
468
|
+
'env_file': env_rel, 'include': args.include or DEFAULT_INCLUDE,
|
|
469
|
+
'exclude': args.exclude or []}
|
|
470
|
+
proj.save_config()
|
|
471
|
+
out(f'wrote {os.path.relpath(proj.config_path, proj.root)}')
|
|
472
|
+
ensure_gitignore(proj.root, [META_DIR + '/'])
|
|
473
|
+
if args.no_pull:
|
|
474
|
+
out('skipped the first pull; run: looker-sync pull')
|
|
475
|
+
return OK
|
|
476
|
+
if interactive() and (ask('Pull the project files into this folder now? (y/n)', 'y') or 'y').lower() != 'y':
|
|
477
|
+
out('skipped the first pull; run: looker-sync pull')
|
|
478
|
+
return OK
|
|
479
|
+
return do_pull(proj, sess, args.force)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def cmd_pull(proj, args):
|
|
483
|
+
return do_pull(proj, open_session(proj, args), args.force, args.dry_run)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def cmd_status(proj, args):
|
|
487
|
+
sess = open_session(proj, args)
|
|
488
|
+
kinds = plan(proj.local_files(), sess.remote_files(), proj.state.get('files', {}))
|
|
489
|
+
show(kinds, 'status')
|
|
490
|
+
info = getattr(sess, 'branch_info', None) or {}
|
|
491
|
+
if sess.mode == 'dev' and info.get('ahead_count') is not None:
|
|
492
|
+
out(f'git: {info.get("ahead_count")} commit(s) ahead of and {info.get("behind_count")} behind '
|
|
493
|
+
f'{info.get("remote_ref") or "the remote"}; commit and push with git, not looker-sync')
|
|
494
|
+
if proj.state.get('synced_at'):
|
|
495
|
+
out(f'last sync: {proj.state["synced_at"]} by {proj.state.get("user")}')
|
|
496
|
+
return OK
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def do_push(proj, sess, force=False, delete=False, dry_run=False):
|
|
500
|
+
if sess.mode != 'dev':
|
|
501
|
+
raise Refused('push is refused in prod mode; switch --mode dev first')
|
|
502
|
+
local, remote = proj.local_files(), sess.remote_files()
|
|
503
|
+
kinds = plan(local, remote, proj.state.get('files', {}))
|
|
504
|
+
show(kinds, 'push plan')
|
|
505
|
+
guarded = [p for p, k in kinds.items() if k in REMOTE_CHANGED]
|
|
506
|
+
if guarded and not force:
|
|
507
|
+
raise Refused('changed in Looker since the last sync: ' + ', '.join(guarded)
|
|
508
|
+
+ '. Pull first, or push --force to overwrite with the local copy.')
|
|
509
|
+
writes = [p for p, k in kinds.items() if p in local and k in ('local-new', 'local-modified', 'conflict', 'remote-deleted')]
|
|
510
|
+
deletes = [p for p, k in kinds.items() if p not in local and k in ('remote-new', 'local-deleted', 'conflict')]
|
|
511
|
+
skipped = [p for p, k in kinds.items() if k == 'remote-modified']
|
|
512
|
+
if skipped:
|
|
513
|
+
out('left alone, unchanged locally (pull to get Looker\'s copy): ' + ', '.join(skipped))
|
|
514
|
+
if deletes and not delete:
|
|
515
|
+
out('not deleted without --delete: ' + ', '.join(deletes))
|
|
516
|
+
deletes = []
|
|
517
|
+
if dry_run:
|
|
518
|
+
out(f'dry run: would write {len(writes)} and delete {len(deletes)} file(s); nothing written')
|
|
519
|
+
return OK
|
|
520
|
+
# New files first: a changed file (an include:) is more likely to reference a new one than the
|
|
521
|
+
# reverse, so stopping halfway leaves Looker valid. Each write is read back and recorded at once,
|
|
522
|
+
# so a failure later in the run never makes the next push see our own writes as Looker edits.
|
|
523
|
+
writes.sort(key=lambda p: p in remote)
|
|
524
|
+
done, verified, bad = [], {}, []
|
|
525
|
+
try:
|
|
526
|
+
for p in writes:
|
|
527
|
+
sess.write(p, local[p], exists=p in remote)
|
|
528
|
+
done.append(p)
|
|
529
|
+
out(f' wrote {p}')
|
|
530
|
+
back = sess.read(p)
|
|
531
|
+
if back == local[p]:
|
|
532
|
+
verified[p] = sha(back)
|
|
533
|
+
record(proj, sess, {p: verified[p]})
|
|
534
|
+
else:
|
|
535
|
+
bad.append(p)
|
|
536
|
+
for p in deletes:
|
|
537
|
+
sess.delete(p)
|
|
538
|
+
done.append(p)
|
|
539
|
+
out(f' deleted {p}')
|
|
540
|
+
record(proj, sess, {}, drops=[p])
|
|
541
|
+
except LookerError:
|
|
542
|
+
out('push stopped partway. Recorded as in sync, so the next push retries only the rest.')
|
|
543
|
+
out(' written: ' + (', '.join(done) or '(none)'))
|
|
544
|
+
out(' not written: ' + (', '.join(p for p in writes + deletes if p not in done) or '(none)'))
|
|
545
|
+
raise
|
|
546
|
+
updates = in_sync(local, remote)
|
|
547
|
+
updates.update(verified)
|
|
548
|
+
record(proj, sess, updates, drops=deletes)
|
|
549
|
+
out(f'read-back: {len(verified)}/{len(writes)} byte-equal')
|
|
550
|
+
for p in bad:
|
|
551
|
+
out(f' MISMATCH {p}: Looker holds different bytes than were sent')
|
|
552
|
+
errors = sess.validate() if (writes or deletes) else 0
|
|
553
|
+
if bad:
|
|
554
|
+
return LOOKER_ERROR
|
|
555
|
+
return INVALID if errors else OK
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def cmd_push(proj, args):
|
|
559
|
+
return do_push(proj, open_session(proj, args), args.force, args.delete, args.dry_run)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def ask(question, choices):
|
|
563
|
+
while True:
|
|
564
|
+
ans = input(f'{question} ').strip().lower()
|
|
565
|
+
for c in choices:
|
|
566
|
+
if ans and c.startswith(ans):
|
|
567
|
+
return c
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def cmd_switch(proj, args):
|
|
571
|
+
cfg = proj.require_config()
|
|
572
|
+
interactive = sys.stdin.isatty()
|
|
573
|
+
sess = open_session(proj, args, check_branch=False)
|
|
574
|
+
target_mode = 'dev' if args.branch else args.mode
|
|
575
|
+
if target_mode == 'dev' and sess.mode != 'dev':
|
|
576
|
+
sess.set_mode('dev')
|
|
577
|
+
sess.branch = sess.current_branch()
|
|
578
|
+
target_branch = args.branch or (sess.branch if target_mode == 'dev' else None)
|
|
579
|
+
if target_mode == cfg['mode'] and (target_mode == 'prod' or target_branch == cfg.get('branch') == sess.branch):
|
|
580
|
+
out(f'already on {target_mode} {target_branch or ""}; nothing to switch')
|
|
581
|
+
return OK
|
|
582
|
+
if args.branch:
|
|
583
|
+
names = [b.get('name') for b in sess.L.call('GET', f'/projects/{cfg["project"]}/git_branches') or []]
|
|
584
|
+
if args.branch not in names:
|
|
585
|
+
raise Refused(f'branch {args.branch!r} does not exist in Looker; switch never creates branches')
|
|
586
|
+
|
|
587
|
+
def remote_guard():
|
|
588
|
+
if cfg['mode'] != 'dev':
|
|
589
|
+
return
|
|
590
|
+
dirty = sess.uncommitted()
|
|
591
|
+
if dirty:
|
|
592
|
+
raise Refused('Looker dev workspace has uncommitted changes:\n'
|
|
593
|
+
+ '\n'.join(f' {p} ({t})' for p, t in dirty)
|
|
594
|
+
+ '\n Commit or revert them in the Looker IDE, then run switch again.')
|
|
595
|
+
|
|
596
|
+
remote_guard()
|
|
597
|
+
local = proj.local_files()
|
|
598
|
+
known = proj.state.get('files', {})
|
|
599
|
+
changed = [p for p in sorted(set(local) | set(known)) if sha(local.get(p)) != known.get(p)]
|
|
600
|
+
if changed:
|
|
601
|
+
out('local changes not pushed to Looker:')
|
|
602
|
+
for p in changed:
|
|
603
|
+
out(f' {p}')
|
|
604
|
+
choice = args.local_changes
|
|
605
|
+
if not choice and interactive:
|
|
606
|
+
choice = ask('[p]ush them first, [b]ack up and discard, or [c]ancel?', ['push', 'backup', 'cancel'])
|
|
607
|
+
if not choice:
|
|
608
|
+
raise Refused('no terminal to ask; pass --local-changes push|backup|cancel')
|
|
609
|
+
if choice == 'cancel':
|
|
610
|
+
raise Refused('cancelled; nothing changed')
|
|
611
|
+
if choice == 'push':
|
|
612
|
+
if cfg['mode'] != 'dev':
|
|
613
|
+
raise Refused('cannot push from prod mode; use --local-changes backup or cancel')
|
|
614
|
+
do_push(proj, sess)
|
|
615
|
+
remote_guard()
|
|
616
|
+
if choice == 'backup':
|
|
617
|
+
stamp = datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
|
|
618
|
+
label = (cfg.get('branch') if cfg['mode'] == 'dev' else 'production') or 'unknown'
|
|
619
|
+
dest = os.path.join(proj.meta, 'backups', f'{re.sub(r"[^A-Za-z0-9._-]", "_", label)}-{stamp}')
|
|
620
|
+
for p, data in local.items():
|
|
621
|
+
full = os.path.join(dest, *p.split('/'))
|
|
622
|
+
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
623
|
+
with open(full, 'wb') as f:
|
|
624
|
+
f.write(data)
|
|
625
|
+
out(f'backed up {len(local)} local file(s) to {os.path.relpath(dest, proj.root)}')
|
|
626
|
+
|
|
627
|
+
where = f'branch {target_branch}' if target_mode == 'dev' else 'production'
|
|
628
|
+
if not args.yes:
|
|
629
|
+
if not interactive:
|
|
630
|
+
raise Refused(f'no terminal to confirm; pass --yes to switch to {where}')
|
|
631
|
+
if ask(f'Switch to {where}? This moves your Looker IDE too. [y/N]', ['yes', 'no']) != 'yes':
|
|
632
|
+
raise Refused('cancelled; nothing changed')
|
|
633
|
+
|
|
634
|
+
if target_mode == 'dev':
|
|
635
|
+
if sess.branch != target_branch:
|
|
636
|
+
sess.L.call('PUT', f'/projects/{cfg["project"]}/git_branch', body={'name': target_branch})
|
|
637
|
+
now = sess.current_branch()
|
|
638
|
+
if now != target_branch:
|
|
639
|
+
raise LookerError(f'checked out {target_branch!r} but Looker reports {now!r}')
|
|
640
|
+
sess.branch = now
|
|
641
|
+
out(f'Looker dev workspace now on {target_branch}; a Looker IDE tab shows it after a refresh')
|
|
642
|
+
else:
|
|
643
|
+
sess.set_mode('prod')
|
|
644
|
+
sess.branch = sess.current_branch()
|
|
645
|
+
|
|
646
|
+
for p in local:
|
|
647
|
+
os.remove(os.path.join(proj.root, *p.split('/')))
|
|
648
|
+
remote = sess.remote_files()
|
|
649
|
+
for p, data in remote.items():
|
|
650
|
+
proj.write_local(p, data)
|
|
651
|
+
cfg.update({'mode': target_mode, 'branch': target_branch})
|
|
652
|
+
proj.save_config()
|
|
653
|
+
proj.state = {'files': {}}
|
|
654
|
+
record(proj, sess, {p: sha(d) for p, d in remote.items()})
|
|
655
|
+
out(f'switched to {target_mode} ({where}); mirrored {len(remote)} file(s), removed {len(local)} old local file(s)')
|
|
656
|
+
return OK
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def version():
|
|
660
|
+
try:
|
|
661
|
+
from importlib.metadata import version as dist_version
|
|
662
|
+
return dist_version('looker-sync')
|
|
663
|
+
except Exception:
|
|
664
|
+
return 'unknown'
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def skill_file():
|
|
668
|
+
here = os.path.dirname(os.path.abspath(__file__))
|
|
669
|
+
for rel in (('_skills_data', 'looker-sync', 'SKILL.md'), # installed wheel
|
|
670
|
+
('..', '..', 'skills', 'looker-sync', 'SKILL.md')): # source checkout
|
|
671
|
+
path = os.path.normpath(os.path.join(here, *rel))
|
|
672
|
+
if os.path.isfile(path):
|
|
673
|
+
return path
|
|
674
|
+
raise LookerError('SKILL.md is not packaged in this install')
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def cmd_skill(args):
|
|
678
|
+
try:
|
|
679
|
+
path = skill_file()
|
|
680
|
+
except LookerError as e:
|
|
681
|
+
out(f'LOOKER ERROR: {e}')
|
|
682
|
+
return LOOKER_ERROR
|
|
683
|
+
if args.path:
|
|
684
|
+
out(path)
|
|
685
|
+
else:
|
|
686
|
+
with open(path, encoding='utf-8') as f:
|
|
687
|
+
sys.stdout.write(f.read())
|
|
688
|
+
return OK
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def main(argv=None):
|
|
692
|
+
ap = argparse.ArgumentParser(
|
|
693
|
+
prog='looker-sync', description=__doc__.split('\n\n')[0],
|
|
694
|
+
epilog='Run `looker-sync skill` for the full guide (workflows, guards, traps) written for agents.')
|
|
695
|
+
ap.add_argument('--version', action='version', version='looker-sync ' + version())
|
|
696
|
+
ap.add_argument('-C', '--project-dir', default='.', help='Looker project folder (default: cwd)')
|
|
697
|
+
ap.add_argument('--env-file', help='env file with LOOKER_URL, LOOKER_CLIENT_ID, LOOKER_CLIENT_SECRET')
|
|
698
|
+
sub = ap.add_subparsers(dest='cmd', required=True)
|
|
699
|
+
p = sub.add_parser('init', help='set up this folder (works when empty), then a first pull')
|
|
700
|
+
p.add_argument('--project', help='Looker project id; asked for on a terminal if omitted')
|
|
701
|
+
p.add_argument('--mode', choices=['dev', 'prod'], default='dev')
|
|
702
|
+
p.add_argument('--branch', help="default: the branch Looker's dev workspace is on")
|
|
703
|
+
p.add_argument('--include', nargs='+')
|
|
704
|
+
p.add_argument('--exclude', nargs='+')
|
|
705
|
+
p.add_argument('--no-pull', action='store_true', help='write the metadata only')
|
|
706
|
+
p.add_argument('--force', action='store_true', help="take Looker's copy over files already in the folder")
|
|
707
|
+
p = sub.add_parser('pull', help='copy Looker files to disk; never deletes local files')
|
|
708
|
+
p.add_argument('--force', action='store_true', help="take Looker's copy over unpushed local changes")
|
|
709
|
+
p.add_argument('--dry-run', action='store_true')
|
|
710
|
+
sub.add_parser('status', help='show what differs; writes nothing')
|
|
711
|
+
p = sub.add_parser('push', help='write local changes, read them back, validate')
|
|
712
|
+
p.add_argument('--force', action='store_true', help='overwrite files changed in Looker since the last sync')
|
|
713
|
+
p.add_argument('--delete', action='store_true', help='delete files that exist only in Looker')
|
|
714
|
+
p.add_argument('--dry-run', action='store_true')
|
|
715
|
+
p = sub.add_parser('switch', help='move Looker and the folder to another branch or mode')
|
|
716
|
+
g = p.add_mutually_exclusive_group(required=True)
|
|
717
|
+
g.add_argument('--branch')
|
|
718
|
+
g.add_argument('--mode', choices=['dev', 'prod'])
|
|
719
|
+
p.add_argument('--local-changes', choices=['push', 'backup', 'cancel'],
|
|
720
|
+
help='what to do with unpushed local changes when there is no terminal')
|
|
721
|
+
p.add_argument('--yes', action='store_true', help='confirm the switch without a terminal')
|
|
722
|
+
p = sub.add_parser('skill', help='print the full usage guide written for agents')
|
|
723
|
+
p.add_argument('--path', action='store_true', help='print where the guide file is instead')
|
|
724
|
+
args = ap.parse_args(argv)
|
|
725
|
+
if args.cmd == 'skill':
|
|
726
|
+
return cmd_skill(args)
|
|
727
|
+
proj = Project(args.project_dir)
|
|
728
|
+
try:
|
|
729
|
+
return {'init': cmd_init, 'pull': cmd_pull, 'status': cmd_status,
|
|
730
|
+
'push': cmd_push, 'switch': cmd_switch}[args.cmd](proj, args)
|
|
731
|
+
except Refused as e:
|
|
732
|
+
out(f'REFUSED: {e}')
|
|
733
|
+
return REFUSED
|
|
734
|
+
except LookerError as e:
|
|
735
|
+
out(f'LOOKER ERROR: {e}')
|
|
736
|
+
return LOOKER_ERROR
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
if __name__ == '__main__':
|
|
740
|
+
sys.exit(main())
|