apify-fingerprint-datapoints 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. apify_fingerprint_datapoints-0.0.1/.editorconfig +12 -0
  2. apify_fingerprint_datapoints-0.0.1/.eslintrc.json +43 -0
  3. apify_fingerprint_datapoints-0.0.1/.github/ISSUE_TEMPLATE/bug_report.md +25 -0
  4. apify_fingerprint_datapoints-0.0.1/.github/ISSUE_TEMPLATE/feature_request.md +17 -0
  5. apify_fingerprint_datapoints-0.0.1/.github/ISSUE_TEMPLATE/question.md +20 -0
  6. apify_fingerprint_datapoints-0.0.1/.github/workflows/check-pr-title.yml +14 -0
  7. apify_fingerprint_datapoints-0.0.1/.github/workflows/e2e-benchmark.yml +49 -0
  8. apify_fingerprint_datapoints-0.0.1/.github/workflows/model-updater.yml +94 -0
  9. apify_fingerprint_datapoints-0.0.1/.github/workflows/release-publish.yml +62 -0
  10. apify_fingerprint_datapoints-0.0.1/.github/workflows/release_python.yaml +26 -0
  11. apify_fingerprint_datapoints-0.0.1/.github/workflows/test-and-sync.yml +89 -0
  12. apify_fingerprint_datapoints-0.0.1/.gitignore +26 -0
  13. apify_fingerprint_datapoints-0.0.1/CONTRIBUTING.md +93 -0
  14. apify_fingerprint_datapoints-0.0.1/LICENSE.md +201 -0
  15. apify_fingerprint_datapoints-0.0.1/PKG-INFO +234 -0
  16. apify_fingerprint_datapoints-0.0.1/README.md +109 -0
  17. apify_fingerprint_datapoints-0.0.1/apify_fingerprint_datapoints/CHANGELOG.md +7 -0
  18. apify_fingerprint_datapoints-0.0.1/apify_fingerprint_datapoints/README.md +7 -0
  19. apify_fingerprint_datapoints-0.0.1/apify_fingerprint_datapoints/__init__.py +26 -0
  20. apify_fingerprint_datapoints-0.0.1/jest.config.js +30 -0
  21. apify_fingerprint_datapoints-0.0.1/package-lock.json +16724 -0
  22. apify_fingerprint_datapoints-0.0.1/package.json +77 -0
  23. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/package.json +40 -0
  24. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/src/constants.ts +2 -0
  25. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/src/data_files/fingerprint-network-definition.zip +0 -0
  26. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/src/fingerprint-generator.ts +274 -0
  27. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/src/index.ts +2 -0
  28. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/tsconfig.build.json +7 -0
  29. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-generator/tsconfig.json +4 -0
  30. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/package.json +52 -0
  31. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/src/constants.ts +1 -0
  32. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/src/fingerprint-injector.ts +332 -0
  33. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/src/index.ts +1 -0
  34. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/src/utils.js +749 -0
  35. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/tsconfig.build.json +7 -0
  36. apify_fingerprint_datapoints-0.0.1/packages/fingerprint-injector/tsconfig.json +4 -0
  37. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/README.md +155 -0
  38. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/package.json +38 -0
  39. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/src/bayesian-network.ts +117 -0
  40. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/src/bayesian-node.ts +200 -0
  41. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/src/index.ts +2 -0
  42. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/src/utils.ts +133 -0
  43. apify_fingerprint_datapoints-0.0.1/packages/generative-bayesian-network/tsconfig.build.json +7 -0
  44. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/package.json +36 -0
  45. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/src/generator-networks-creator.ts +259 -0
  46. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/src/index.ts +1 -0
  47. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/src/network_structures/fingerprint-network-structure.zip +0 -0
  48. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/src/network_structures/header-network-structure.zip +0 -0
  49. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/src/network_structures/input-network-structure.zip +0 -0
  50. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/tsconfig.build.json +7 -0
  51. apify_fingerprint_datapoints-0.0.1/packages/generator-networks-creator/tsconfig.json +4 -0
  52. apify_fingerprint_datapoints-0.0.1/packages/header-generator/README.md +144 -0
  53. apify_fingerprint_datapoints-0.0.1/packages/header-generator/package.json +45 -0
  54. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/constants.ts +28 -0
  55. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/data_files/browser-helper-file.json +1 -0
  56. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/data_files/header-network-definition.zip +0 -0
  57. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/data_files/headers-order.json +117 -0
  58. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/data_files/input-network-definition.zip +0 -0
  59. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/header-generator.ts +517 -0
  60. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/index.ts +2 -0
  61. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/presets.ts +59 -0
  62. apify_fingerprint_datapoints-0.0.1/packages/header-generator/src/utils.ts +91 -0
  63. apify_fingerprint_datapoints-0.0.1/packages/header-generator/tsconfig.build.json +7 -0
  64. apify_fingerprint_datapoints-0.0.1/packages/header-generator/tsconfig.json +4 -0
  65. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/Dockerfile +17 -0
  66. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/cert/localhost-cert.pem +22 -0
  67. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/cert/localhost-privkey.pem +28 -0
  68. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/collector.js +65 -0
  69. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/package.json +22 -0
  70. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/server.js +41 -0
  71. apify_fingerprint_datapoints-0.0.1/packages/header-order-collector/serverv2.js +41 -0
  72. apify_fingerprint_datapoints-0.0.1/pyproject.toml +49 -0
  73. apify_fingerprint_datapoints-0.0.1/renovate.json +27 -0
  74. apify_fingerprint_datapoints-0.0.1/scripts/copy.ts +70 -0
  75. apify_fingerprint_datapoints-0.0.1/scripts/netgen.sh +15 -0
  76. apify_fingerprint_datapoints-0.0.1/scripts/netgen.ts +16 -0
  77. apify_fingerprint_datapoints-0.0.1/scripts/publish.sh +17 -0
  78. apify_fingerprint_datapoints-0.0.1/scripts/typescript_fixes.mjs +36 -0
  79. apify_fingerprint_datapoints-0.0.1/static/logo_big_dark.svg +309 -0
  80. apify_fingerprint_datapoints-0.0.1/static/logo_big_light.svg +309 -0
  81. apify_fingerprint_datapoints-0.0.1/test/antibot-services/antibot.js +57 -0
  82. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/README.md +4 -0
  83. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/cloudflare-websites.csv +139 -0
  84. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/cloudflare.ts +128 -0
  85. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/engine/combo.ts +36 -0
  86. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/engine/fingerprint-suite.ts +36 -0
  87. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/engine/playwright-extra.ts +26 -0
  88. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/engine/vanilla-playwright.ts +51 -0
  89. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/report.html +87 -0
  90. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/report.png +0 -0
  91. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/speed.js +43 -0
  92. apify_fingerprint_datapoints-0.0.1/test/antibot-services/live-testing/utils/generateReport.ts +83 -0
  93. apify_fingerprint_datapoints-0.0.1/test/fingerprint-generator/generation.test.ts +146 -0
  94. apify_fingerprint_datapoints-0.0.1/test/fingerprint-injector/crossbrowser.test.ts +62 -0
  95. apify_fingerprint_datapoints-0.0.1/test/fingerprint-injector/fingerprint-injector.test.ts +460 -0
  96. apify_fingerprint_datapoints-0.0.1/test/fingerprint-injector/test.html +7 -0
  97. apify_fingerprint_datapoints-0.0.1/test/generative-bayesian-network/generation.test.ts +68 -0
  98. apify_fingerprint_datapoints-0.0.1/test/generative-bayesian-network/testDataset.csv +2751 -0
  99. apify_fingerprint_datapoints-0.0.1/test/generative-bayesian-network/testNetworkDefinition.zip +0 -0
  100. apify_fingerprint_datapoints-0.0.1/test/generative-bayesian-network/testNetworkStructureDefinition.zip +0 -0
  101. apify_fingerprint_datapoints-0.0.1/test/generator-networks-creator/generator-networks-creator.test.ts +145 -0
  102. apify_fingerprint_datapoints-0.0.1/test/header-generator/browser-list.test.ts +16 -0
  103. apify_fingerprint_datapoints-0.0.1/test/header-generator/generation.test.ts +167 -0
  104. apify_fingerprint_datapoints-0.0.1/test/header-generator/presets.test.ts +11 -0
  105. apify_fingerprint_datapoints-0.0.1/test/tsconfig.json +13 -0
  106. apify_fingerprint_datapoints-0.0.1/tsconfig.build.json +22 -0
  107. apify_fingerprint_datapoints-0.0.1/tsconfig.eslint.json +15 -0
  108. apify_fingerprint_datapoints-0.0.1/tsconfig.json +13 -0
  109. apify_fingerprint_datapoints-0.0.1/turbo.json +24 -0
@@ -0,0 +1,12 @@
1
+ root = true
2
+
3
+ [*]
4
+ indent_style = space
5
+ indent_size = 4
6
+ charset = utf-8
7
+ trim_trailing_whitespace = true
8
+ insert_final_newline = true
9
+ end_of_line = lf
10
+ # editorconfig-tools is unable to ignore longs strings or urls
11
+ max_line_length = null
12
+ quote_type = single
@@ -0,0 +1,43 @@
1
+ {
2
+ "root": true,
3
+ "env": {
4
+ "browser": true,
5
+ "es2020": true,
6
+ "node": true
7
+ },
8
+ "extends": [
9
+ "@apify/eslint-config-ts"
10
+ ],
11
+ "parserOptions": {
12
+ "project": "./tsconfig.eslint.json",
13
+ "ecmaVersion": 2020
14
+ },
15
+ "plugins": [
16
+ "@typescript-eslint"
17
+ ],
18
+ "ignorePatterns": [
19
+ "node_modules",
20
+ "dist",
21
+ "coverage",
22
+ "**/*.js",
23
+ "**/*.d.ts"
24
+ ],
25
+ "rules": {
26
+ "no-void": 0,
27
+ "no-underscore-dangle": 0,
28
+ "max-classes-per-file": 0,
29
+ "no-console": "error",
30
+ "@typescript-eslint/array-type": "error",
31
+ "@typescript-eslint/no-floating-promises": "error",
32
+ "@typescript-eslint/ban-ts-comment": 0,
33
+ "@typescript-eslint/no-unused-vars": "off",
34
+ "@typescript-eslint/no-empty-interface": "off",
35
+ "@typescript-eslint/no-explicit-any": "off",
36
+ "@typescript-eslint/no-empty-function": "off",
37
+ "import/no-extraneous-dependencies": "off",
38
+ "@typescript-eslint/member-delimiter-style": ["error", {
39
+ "multiline": { "delimiter": "semi", "requireLast": true },
40
+ "singleline": { "delimiter": "semi", "requireLast": false }
41
+ }]
42
+ }
43
+ }
@@ -0,0 +1,25 @@
1
+ ---
2
+ name: Bug report
3
+ about: Create a report to help us improve
4
+ title: ''
5
+ labels: bug
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Describe the bug**
11
+ A clear and concise description of what the bug is.
12
+
13
+ **To Reproduce**
14
+ Add an example of code that produces the problem you're reporting.
15
+ We should be able to copy your code, run it and get the same result.
16
+
17
+ **Expected behavior**
18
+ A clear and concise description of what you expected to happen.
19
+
20
+ **System information:**
21
+ - OS: [e.g. MacOS]
22
+ - Node.js version [e.g. 14.7]
23
+
24
+ **Additional context**
25
+ Add any other context about the problem here.
@@ -0,0 +1,17 @@
1
+ ---
2
+ name: Feature request
3
+ about: For new things and improvements that are not bugs.
4
+ title: ''
5
+ labels: feature
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Describe the feature**
11
+ A clear and concise description of what feature you're requesting.
12
+
13
+ **Motivation**
14
+ Why would you like to have this feature in the SDK. How would you use it? How would it help you?
15
+
16
+ **Constraints**
17
+ Are there any requirements or limits to the feature that need to be satisfied?
@@ -0,0 +1,20 @@
1
+ ---
2
+ name: Question
3
+ about: When you need help with something.
4
+ title: ''
5
+ labels: question
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ Please submit new issues for new feature requests and bug reports only.
11
+ For questions please join Apify Discord. You could get help from Apify team members and the community there.
12
+
13
+ - [Discord](https://discord.gg/jyEM2PRvMU)
14
+
15
+ Please also make sure that you visit all the relevant information sources and use search there.
16
+
17
+ - [SDK Documentation](https://sdk.apify.com)
18
+ - [Our Stack Overflow](https://stackoverflow.com/tags/apify/)
19
+ - [Apify Platform Docs](https://docs.apify.com)
20
+ - [Apify Help](https://help.apify.com)
@@ -0,0 +1,14 @@
1
+ name: Check PR title
2
+
3
+ on:
4
+ pull_request_target:
5
+ types: [ opened, edited, synchronize ]
6
+
7
+ jobs:
8
+ check_pr_title:
9
+ name: 'Check PR title'
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: amannn/action-semantic-pull-request@v5.5.3
13
+ env:
14
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -0,0 +1,49 @@
1
+ name: Run Benchmarks
2
+
3
+ on:
4
+ # Triggers the workflow every Monday at 5am
5
+ schedule:
6
+ - cron: "0 5 * * 1"
7
+ workflow_dispatch:
8
+
9
+
10
+ jobs:
11
+ run_benchmark:
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ with:
17
+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
18
+ - name: Use Node.js 22
19
+ uses: actions/setup-node@v4
20
+ with:
21
+ node-version: 22
22
+
23
+ - name: Cache Node Modules
24
+ uses: actions/cache@v4
25
+ with:
26
+ path: |
27
+ node_modules
28
+ key: cache-${{ github.run_id }}
29
+
30
+ - name: Install Dependencies
31
+ run: npm ci
32
+
33
+ - name: Install browsers
34
+ run: npx playwright install
35
+
36
+ - name: Build the project
37
+ run: npm run build
38
+
39
+ - name: Run benchmarks
40
+ run: npm run benchmark
41
+
42
+ - name: Update the benchmark report
43
+ uses: stefanzweifel/git-auto-commit-action@v5
44
+ with:
45
+ commit_message: "chore: [skip-ci] update benchmark results"
46
+ branch: master
47
+ file_pattern: '**/report.png'
48
+ commit_user_name: "Mark Bench[bot] 💪🏼"
49
+ commit_user_email: markbench[bot]@users.noreply.github.com
@@ -0,0 +1,94 @@
1
+ name: Model updater
2
+
3
+ on:
4
+ # Triggers the workflow every month
5
+ # schedule:
6
+ # - cron: "0 0 1 * *"
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ build_model:
11
+ # The type of runner that the job will run on
12
+ runs-on: ubuntu-latest
13
+
14
+ # Steps represent a sequence of tasks that will be executed as part of the job
15
+ steps:
16
+ # Runs a single command using the runners shell
17
+ - uses: actions/checkout@v4
18
+ with:
19
+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
20
+ - name: Use Node.js 22
21
+ uses: actions/setup-node@v4
22
+ with:
23
+ node-version: 22
24
+
25
+ - name: Cache node modules + browsers
26
+ uses: actions/cache@v4
27
+ with:
28
+ path: |
29
+ ~/.npm
30
+ ~/.cache/ms-playwright
31
+ key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
32
+
33
+ - name: Install Dependencies
34
+ run: npm ci
35
+
36
+ - name: Install Playwright deps
37
+ run: npx playwright install
38
+
39
+ - name: Install xvfb
40
+ run: |
41
+ sudo apt-get update
42
+ sudo apt-get -y install xvfb
43
+
44
+ - name: Install Playwright browsers
45
+ run: |
46
+ npx playwright install webkit
47
+ npx playwright install chrome
48
+ npx playwright install msedge
49
+
50
+ - name: Install Playwright deps v2
51
+ run: npx playwright install-deps
52
+
53
+ - name: Collect HTTP header orders
54
+ run: |
55
+ xvfb-run -- node ./packages/header-order-collector/collector.js | tee ./packages/header-generator/src/data_files/headers-order.json
56
+
57
+ - name: Generate model and update versions
58
+ run: |
59
+ npm version patch --git-tag-version=false
60
+ export GIT_TAG=$(node -p "require('./package.json').version")
61
+
62
+ npm run prerelease
63
+ npm run buildNetwork
64
+ env:
65
+ APIFY_FINGERPRINT_DATASET_ID: ${{ secrets.APIFY_FINGERPRINT_DATASET_ID }}
66
+
67
+ - name: Get the new version number
68
+ id: get_version
69
+ run: |
70
+ echo "::set-output name=version::$(node -p "require('./package.json').version")"
71
+
72
+ - name: Push new model
73
+ uses: stefanzweifel/git-auto-commit-action@v5
74
+ with:
75
+ commit_message: "[auto] fingerprint/header model update"
76
+ branch: master
77
+ tagging_message: "v${{ steps.get_version.outputs.version }}"
78
+
79
+ file_pattern: "./packages/ ./package.json ./package-lock.json"
80
+
81
+ commit_user_name: "modelmaker[bot] ⚒️"
82
+ commit_user_email: modelmaker[bot]@users.noreply.github.com
83
+
84
+ - name: Create release
85
+ uses: actions/create-release@v1
86
+ env:
87
+ GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
88
+ with:
89
+ tag_name: "v${{ steps.get_version.outputs.version }}"
90
+ release_name: "v${{ steps.get_version.outputs.version }}"
91
+ body: |
92
+ This is an automated release of the model used by the `fingerprint-suite`.
93
+
94
+ The only difference between this release and the previous one is the model used to generate fingerprints.
@@ -0,0 +1,62 @@
1
+ name: Publish
2
+ on:
3
+ release:
4
+ types: [published]
5
+
6
+ jobs:
7
+ test_and_sync:
8
+ uses: apify/fingerprint-suite/.github/workflows/test-and-sync.yml@master
9
+
10
+ publish_latest:
11
+ name: Publish latest
12
+ runs-on: ubuntu-latest
13
+ needs: [ test_and_sync ]
14
+ steps:
15
+ - name: Checkout Source code
16
+ uses: actions/checkout@v4
17
+ with:
18
+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
19
+
20
+ - name: Use Node.js 22
21
+ uses: actions/setup-node@v4
22
+ with:
23
+ node-version: 22
24
+ cache: 'npm'
25
+ cache-dependency-path: 'package-lock.json'
26
+
27
+ - name: Turbo cache
28
+ id: turbo-cache
29
+ uses: actions/cache@v4
30
+ with:
31
+ path: .turbo
32
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
33
+ restore-keys: |
34
+ turbo-${{ github.job }}-${{ github.ref_name }}-
35
+
36
+ - name: Install Dependencies
37
+ run: npm ci
38
+
39
+ - name: Release latest versions
40
+ run: |
41
+ git config --global user.name 'Apify Release Bot'
42
+ git config --global user.email 'noreply@apify.com'
43
+
44
+ echo "access=public" >> ~/.npmrc
45
+ echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
46
+
47
+ export GIT_TAG=$(echo $GITHUB_REF | sed 's/refs\/tags\///g')
48
+ npm run release
49
+ env:
50
+ NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}
51
+ GIT_USER: "noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"
52
+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
53
+
54
+ - name: Commit the version bump
55
+ uses: stefanzweifel/git-auto-commit-action@v5
56
+ with:
57
+ branch: master
58
+ commit_message: 'chore: [skip ci] sync new package.json versions'
59
+ file_pattern: 'package*.json'
60
+ commit_user_name: Apify Bot
61
+ commit_user_email: my-github-actions-bot@example.org
62
+ commit_author: Apify Bot <apify@apify.com>
@@ -0,0 +1,26 @@
1
+ name: Create a release
2
+ on: workflow_dispatch
3
+
4
+ jobs:
5
+ publish_to_pypi:
6
+ name: Publish to PyPI
7
+ runs-on: ubuntu-latest
8
+ permissions:
9
+ contents: write
10
+ id-token: write # Required for OIDC authentication.
11
+ environment:
12
+ name: pypi
13
+ url: https://pypi.org/project/crawlee
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: actions/setup-python@v5
17
+ with:
18
+ python-version: '3.13'
19
+ - uses: astral-sh/setup-uv@v5
20
+ - name: Build project
21
+ shell: bash
22
+ run: uv build
23
+
24
+ # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication.
25
+ - name: Publish package to PyPI
26
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,89 @@
1
+ name: Run tests
2
+
3
+ on:
4
+ # Push to master will deploy a dev version, push to stable deploys the latest version (and syncs the master branch)
5
+ push:
6
+ branches: [ master, stable ]
7
+ # PRs will run tests (skip deployment)
8
+ pull_request:
9
+ branches: [ master, stable ]
10
+ # A release via GitHub releases will deploy the latest version
11
+ workflow_call:
12
+ workflow_dispatch:
13
+
14
+ jobs:
15
+ lint:
16
+ name: Lint
17
+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
18
+ runs-on: ubuntu-latest
19
+
20
+ steps:
21
+ - uses: actions/checkout@v4
22
+ - name: Use Node.js 22
23
+ uses: actions/setup-node@v4
24
+ with:
25
+ node-version: 22
26
+
27
+ - name: Install Dependencies
28
+ run: npm ci
29
+
30
+ - name: Load Cache
31
+ uses: actions/cache@v4
32
+ with:
33
+ path: |
34
+ node_modules
35
+ key: cache-${{ github.run_id }}
36
+
37
+ - run: npm run lint
38
+
39
+ build_and_test:
40
+ name: Build & Test
41
+ needs: [lint]
42
+ runs-on: ${{ matrix.os }}
43
+
44
+ strategy:
45
+ matrix:
46
+ os: [ ubuntu-latest ]
47
+ node-version: [ 18, 20, 22 ]
48
+
49
+ steps:
50
+ - uses: actions/checkout@v4
51
+ - name: Use Node.js ${{ matrix.node-version }}
52
+ uses: actions/setup-node@v4
53
+ with:
54
+ node-version: ${{ matrix.node-version }}
55
+
56
+ - name: Cache Node Modules
57
+ uses: actions/cache@v4
58
+ with:
59
+ path: |
60
+ node_modules
61
+ key: cache-${{ github.run_id }}
62
+
63
+ - name: Install Playwright deps
64
+ run : npx playwright install --with-deps
65
+
66
+ - name: Install Puppeteer deps
67
+ run : |
68
+ npx puppeteer browsers install chrome
69
+ npx puppeteer browsers install firefox@beta
70
+
71
+ - name: Turbo cache
72
+ id: turbo-cache
73
+ uses: actions/cache@v4
74
+ with:
75
+ path: .turbo
76
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
77
+ restore-keys: |
78
+ turbo-${{ github.job }}-${{ github.ref_name }}-
79
+
80
+ - name: Build
81
+ run: npm run build
82
+
83
+ - name: Run Windows or MacOs tests
84
+ run: npm test
85
+ if: ${{ matrix.os == 'windows-latest' || matrix.os == 'macos-latest' }}
86
+
87
+ - name: Run Linux tests
88
+ run: xvfb-run --auto-servernum -- npm test
89
+ if: ${{ matrix.os == 'ubuntu-latest'}}
@@ -0,0 +1,26 @@
1
+ build
2
+ dist
3
+ build-docs
4
+ node_modules
5
+ *.log
6
+ *.pid
7
+ *.seed
8
+ .DS_Store
9
+ lib
10
+ coverage
11
+ .nyc_output
12
+ logs
13
+ pids
14
+ .idea
15
+ .vscode
16
+ yarn.lock
17
+ tmp
18
+ jsconfig.json
19
+ types
20
+ docs/api
21
+ docs/typedefs
22
+ .history
23
+ .docusaurus
24
+ tsconfig.tsbuildinfo
25
+ .turbo
26
+ .npmrc
@@ -0,0 +1,93 @@
1
+ # Contributing
2
+
3
+ When contributing to this repository, please first discuss the change you wish to make via issue,
4
+ email, or any other method with the owners of this repository before making a change.
5
+
6
+ Please note we have a code of conduct, please follow it in all your interactions with the project.
7
+
8
+ ## Pull Request Process
9
+
10
+ 1. Ensure any install or build dependencies are removed before the end of the layer when doing a
11
+ build.
12
+ 2. Update the README.md and CHANGELOG.md with details of changes to the interface, this includes new environment
13
+ variables, exposed ports, useful file locations and container parameters.
14
+ 3. Increase the version numbers in any examples files and the README.md to the new version that this
15
+ Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/).
16
+ 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you
17
+ do not have permission to do that, you may request the second reviewer to merge it for you.
18
+
19
+ ## Code of Conduct
20
+
21
+ ### Our Pledge
22
+
23
+ In the interest of fostering an open and welcoming environment, we as
24
+ contributors and maintainers pledge to making participation in our project and
25
+ our community a harassment-free experience for everyone, regardless of age, body
26
+ size, disability, ethnicity, gender identity and expression, level of experience,
27
+ nationality, personal appearance, race, religion, or sexual identity and
28
+ orientation.
29
+
30
+ ### Our Standards
31
+
32
+ Examples of behavior that contributes to creating a positive environment
33
+ include:
34
+
35
+ * Using welcoming and inclusive language
36
+ * Being respectful of differing viewpoints and experiences
37
+ * Gracefully accepting constructive criticism
38
+ * Focusing on what is best for the community
39
+ * Showing empathy towards other community members
40
+
41
+ Examples of unacceptable behavior by participants include:
42
+
43
+ * The use of sexualized language or imagery and unwelcome sexual attention or
44
+ advances
45
+ * Trolling, insulting/derogatory comments, and personal or political attacks
46
+ * Public or private harassment
47
+ * Publishing others' private information, such as a physical or electronic
48
+ address, without explicit permission
49
+ * Other conduct which could reasonably be considered inappropriate in a
50
+ professional setting
51
+
52
+ ### Our Responsibilities
53
+
54
+ Project maintainers are responsible for clarifying the standards of acceptable
55
+ behavior and are expected to take appropriate and fair corrective action in
56
+ response to any instances of unacceptable behavior.
57
+
58
+ Project maintainers have the right and responsibility to remove, edit, or
59
+ reject comments, commits, code, wiki edits, issues, and other contributions
60
+ that are not aligned to this Code of Conduct, or to ban temporarily or
61
+ permanently any contributor for other behaviors that they deem inappropriate,
62
+ threatening, offensive, or harmful.
63
+
64
+ ### Scope
65
+
66
+ This Code of Conduct applies both within project spaces and in public spaces
67
+ when an individual is representing the project or its community. Examples of
68
+ representing a project or community include using an official project e-mail
69
+ address, posting via an official social media account, or acting as an appointed
70
+ representative at an online or offline event. Representation of a project may be
71
+ further defined and clarified by project maintainers.
72
+
73
+ ### Enforcement
74
+
75
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
76
+ reported by contacting the project team at support@apify.com. All
77
+ complaints will be reviewed and investigated and will result in a response that
78
+ is deemed necessary and appropriate to the circumstances. The project team is
79
+ obligated to maintain confidentiality with regard to the reporter of an incident.
80
+ Further details of specific enforcement policies may be posted separately.
81
+
82
+ Project maintainers who do not follow or enforce the Code of Conduct in good
83
+ faith may face temporary or permanent repercussions as determined by other
84
+ members of the project's leadership.
85
+
86
+ ### Attribution
87
+
88
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
89
+ available at [http://contributor-covenant.org/version/1/4][version],
90
+ and from [PurpleBooth](https://gist.github.com/PurpleBooth/b24679402957c63ec426).
91
+
92
+ [homepage]: http://contributor-covenant.org
93
+ [version]: http://contributor-covenant.org/version/1/4/