dalla-data-processing 0.0.3__tar.gz → 0.0.10__tar.gz

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/.github/workflows/ci.yml

@@ -55,8 +55,8 @@ jobs:
         include:
           - os: ubuntu-latest
             artifact_name: onion-linux-x86_64
-          - os: macos-latest
-            artifact_name: onion-darwin-x86_64
+          - os: macos-14
+            artifact_name: onion-darwin-universal
     
     steps:
     - uses: actions/checkout@v4

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/.github/workflows/release.yml

@@ -17,8 +17,8 @@ jobs:
         include:
           - os: ubuntu-latest
             artifact_name: onion-linux-x86_64
-          - os: macos-latest
-            artifact_name: onion-darwin-x86_64
+          - os: macos-14
+            artifact_name: onion-darwin-universal
     
     steps:
     - uses: actions/checkout@v4
@@ -46,19 +46,10 @@ jobs:
         path: dalla_data_processing/deduplication/bin/onion-*
         if-no-files-found: error
 
-  build-wheels:
-    name: Build wheels on ${{ matrix.os }}
+  build-wheel:
+    name: Build universal wheel with all platform binaries
     needs: build-onion
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            onion_artifact: onion-linux-x86_64
-          - os: macos-latest
-            onion_artifact: onion-darwin-x86_64
-          - os: windows-latest
-            onion_artifact: none
+    runs-on: ubuntu-latest
     
     steps:
     - uses: actions/checkout@v4
@@ -67,24 +58,24 @@ jobs:
     
     - name: Create bin directory
       run: mkdir -p dalla_data_processing/deduplication/bin
-      shell: bash
     
-    - name: Download Onion binary
-      if: matrix.onion_artifact != 'none'
+    - name: Download Linux binary
+      uses: actions/download-artifact@v4
+      with:
+        name: onion-linux-x86_64
+        path: dalla_data_processing/deduplication/bin/
+    
+    - name: Download macOS universal binary
       uses: actions/download-artifact@v4
       with:
-        name: ${{ matrix.onion_artifact }}
+        name: onion-darwin-universal
         path: dalla_data_processing/deduplication/bin/
     
     - name: Set binary permissions
-      if: matrix.onion_artifact != 'none'
       run: chmod +x dalla_data_processing/deduplication/bin/onion-*
-      shell: bash
     
-    - name: List binary files
-      if: matrix.onion_artifact != 'none'
+    - name: List all binaries
       run: ls -lah dalla_data_processing/deduplication/bin/
-      shell: bash
     
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -94,20 +85,23 @@ jobs:
     - name: Install build dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install build twine
+        pip install build "twine>=5.0,<6.1"
     
     - name: Build wheel
       run: python -m build
     
     - name: List wheel contents
       run: |
-        python -m zipfile -l dist/*.whl | head -50
+        echo "=== Wheel contents (binaries) ==="
+        python -m zipfile -l dist/*.whl | grep onion-
+        echo "=== Wheel info ==="
+        ls -lh dist/
     
-    - name: Upload wheels
+    - name: Upload wheel
       uses: actions/upload-artifact@v4
       with:
-        name: wheels-${{ matrix.os }}
-        path: dist/*
+        name: wheel
+        path: dist/*.whl
 
   build-sdist:
     name: Build source distribution
@@ -136,15 +130,15 @@ jobs:
 
   publish-pypi:
     name: Publish to PyPI
-    needs: [build-wheels, build-sdist]
+    needs: [build-wheel, build-sdist]
     runs-on: ubuntu-latest
     environment: release
     
     steps:
-    # Only use Linux wheel to avoid conflicts
+    # Download the universal wheel with all platform binaries
     - uses: actions/download-artifact@v4
       with:
-        name: wheels-ubuntu-latest
+        name: wheel
         path: dist/
     
     - uses: actions/download-artifact@v4
@@ -157,7 +151,7 @@ jobs:
     
     - name: Verify wheel integrity
       run: |
-        pip install twine
+        pip install "twine>=5.0,<6.1"
         twine check dist/*
     
     - name: Publish to PyPI
@@ -168,7 +162,7 @@ jobs:
 
   create-release:
     name: Create GitHub Release
-    needs: [build-wheels, build-sdist]
+    needs: [build-wheel, build-sdist]
     runs-on: ubuntu-latest
     
     steps:
@@ -178,8 +172,7 @@ jobs:
     
     - uses: actions/download-artifact@v4
       with:
-        pattern: wheels-*
-        merge-multiple: true
+        name: wheel
         path: dist/
     
     - uses: actions/download-artifact@v4

dalla_data_processing-0.0.10/LICENSE

@@ -0,0 +1,353 @@
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright and
+certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+Considerations for licensors: Our public licenses are intended for use
+by those authorized to give the public permission to use material in
+ways otherwise restricted by copyright and certain other rights. Our
+licenses are irrevocable. Licensors should read and understand the terms
+and conditions of the license they choose before applying it. Licensors
+should also secure all rights necessary before applying our licenses so
+that the public can reuse the material as expected. Licensors should
+clearly mark any material not subject to the license. This includes
+other CC-licensed material, or material used under an exception or
+limitation to copyright. More considerations for licensors :
+wiki.creativecommons.org/Considerations\_for\_licensors
+
+Considerations for the public: By using one of our public licenses, a
+licensor grants the public permission to use the licensed material under
+specified terms and conditions. If the licensor's permission is not
+necessary for any reason–for example, because of any applicable
+exception or limitation to copyright–then that use is not regulated by
+the license. Our licenses grant only permissions under copyright and
+certain other rights that a licensor has authority to grant. Use of the
+licensed material may still be restricted for other reasons, including
+because others have copyright or other rights in the material. A
+licensor may make special requests, such as asking that all changes be
+marked or described. Although not required by our licenses, you are
+encouraged to respect those requests where reasonable. More
+considerations for the public :
+wiki.creativecommons.org/Considerations\_for\_licensees
+
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
+Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-ShareAlike 4.0 International Public License
+("Public License"). To the extent this Public License may be interpreted
+as a contract, You are granted the Licensed Rights in consideration of
+Your acceptance of these terms and conditions, and the Licensor grants
+You such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and conditions.
+
+Section 1 – Definitions.
+
+-   a. Adapted Material means material subject to Copyright and Similar
+    Rights that is derived from or based upon the Licensed Material and
+    in which the Licensed Material is translated, altered, arranged,
+    transformed, or otherwise modified in a manner requiring permission
+    under the Copyright and Similar Rights held by the Licensor. For
+    purposes of this Public License, where the Licensed Material is a
+    musical work, performance, or sound recording, Adapted Material is
+    always produced where the Licensed Material is synched in timed
+    relation with a moving image.
+-   b. Adapter's License means the license You apply to Your Copyright
+    and Similar Rights in Your contributions to Adapted Material in
+    accordance with the terms and conditions of this Public License.
+-   c. BY-NC-SA Compatible License means a license listed at
+    creativecommons.org/compatiblelicenses, approved by Creative Commons
+    as essentially the equivalent of this Public License.
+-   d. Copyright and Similar Rights means copyright and/or similar
+    rights closely related to copyright including, without limitation,
+    performance, broadcast, sound recording, and Sui Generis Database
+    Rights, without regard to how the rights are labeled or categorized.
+    For purposes of this Public License, the rights specified in Section
+    2(b)(1)-(2) are not Copyright and Similar Rights.
+-   e. Effective Technological Measures means those measures that, in
+    the absence of proper authority, may not be circumvented under laws
+    fulfilling obligations under Article 11 of the WIPO Copyright Treaty
+    adopted on December 20, 1996, and/or similar international
+    agreements.
+-   f. Exceptions and Limitations means fair use, fair dealing, and/or
+    any other exception or limitation to Copyright and Similar Rights
+    that applies to Your use of the Licensed Material.
+-   g. License Elements means the license attributes listed in the name
+    of a Creative Commons Public License. The License Elements of this
+    Public License are Attribution, NonCommercial, and ShareAlike.
+-   h. Licensed Material means the artistic or literary work, database,
+    or other material to which the Licensor applied this Public License.
+-   i. Licensed Rights means the rights granted to You subject to the
+    terms and conditions of this Public License, which are limited to
+    all Copyright and Similar Rights that apply to Your use of the
+    Licensed Material and that the Licensor has authority to license.
+-   j. Licensor means the individual(s) or entity(ies) granting rights
+    under this Public License.
+-   k. NonCommercial means not primarily intended for or directed
+    towards commercial advantage or monetary compensation. For purposes
+    of this Public License, the exchange of the Licensed Material for
+    other material subject to Copyright and Similar Rights by digital
+    file-sharing or similar means is NonCommercial provided there is no
+    payment of monetary compensation in connection with the exchange.
+-   l. Share means to provide material to the public by any means or
+    process that requires permission under the Licensed Rights, such as
+    reproduction, public display, public performance, distribution,
+    dissemination, communication, or importation, and to make material
+    available to the public including in ways that members of the public
+    may access the material from a place and at a time individually
+    chosen by them.
+-   m. Sui Generis Database Rights means rights other than copyright
+    resulting from Directive 96/9/EC of the European Parliament and of
+    the Council of 11 March 1996 on the legal protection of databases,
+    as amended and/or succeeded, as well as other essentially equivalent
+    rights anywhere in the world.
+-   n. You means the individual or entity exercising the Licensed Rights
+    under this Public License. Your has a corresponding meaning.
+
+Section 2 – Scope.
+
+-   a. License grant.
+    -   1. Subject to the terms and conditions of this Public License,
+        the Licensor hereby grants You a worldwide, royalty-free,
+        non-sublicensable, non-exclusive, irrevocable license to
+        exercise the Licensed Rights in the Licensed Material to:
+        -   A. reproduce and Share the Licensed Material, in whole or in
+            part, for NonCommercial purposes only; and
+        -   B. produce, reproduce, and Share Adapted Material for
+            NonCommercial purposes only.
+    -   2. Exceptions and Limitations. For the avoidance of doubt, where
+        Exceptions and Limitations apply to Your use, this Public
+        License does not apply, and You do not need to comply with its
+        terms and conditions.
+    -   3. Term. The term of this Public License is specified in Section
+        6(a).
+    -   4. Media and formats; technical modifications allowed. The
+        Licensor authorizes You to exercise the Licensed Rights in all
+        media and formats whether now known or hereafter created, and to
+        make technical modifications necessary to do so. The Licensor
+        waives and/or agrees not to assert any right or authority to
+        forbid You from making technical modifications necessary to
+        exercise the Licensed Rights, including technical modifications
+        necessary to circumvent Effective Technological Measures. For
+        purposes of this Public License, simply making modifications
+        authorized by this Section 2(a)(4) never produces Adapted
+        Material.
+    -   5. Downstream recipients.
+        -   A. Offer from the Licensor – Licensed Material. Every
+            recipient of the Licensed Material automatically receives an
+            offer from the Licensor to exercise the Licensed Rights
+            under the terms and conditions of this Public License.
+        -   B. Additional offer from the Licensor – Adapted Material.
+            Every recipient of Adapted Material from You automatically
+            receives an offer from the Licensor to exercise the Licensed
+            Rights in the Adapted Material under the conditions of the
+            Adapter's License You apply.
+        -   C. No downstream restrictions. You may not offer or impose
+            any additional or different terms or conditions on, or apply
+            any Effective Technological Measures to, the Licensed
+            Material if doing so restricts exercise of the Licensed
+            Rights by any recipient of the Licensed Material.
+    -   6. No endorsement. Nothing in this Public License constitutes or
+        may be construed as permission to assert or imply that You are,
+        or that Your use of the Licensed Material is, connected with, or
+        sponsored, endorsed, or granted official status by, the Licensor
+        or others designated to receive attribution as provided in
+        Section 3(a)(1)(A)(i).
+-   b. Other rights.
+    -   1. Moral rights, such as the right of integrity, are not
+        licensed under this Public License, nor are publicity, privacy,
+        and/or other similar personality rights; however, to the extent
+        possible, the Licensor waives and/or agrees not to assert any
+        such rights held by the Licensor to the limited extent necessary
+        to allow You to exercise the Licensed Rights, but not otherwise.
+    -   2. Patent and trademark rights are not licensed under this
+        Public License.
+    -   3. To the extent possible, the Licensor waives any right to
+        collect royalties from You for the exercise of the Licensed
+        Rights, whether directly or through a collecting society under
+        any voluntary or waivable statutory or compulsory licensing
+        scheme. In all other cases the Licensor expressly reserves any
+        right to collect such royalties, including when the Licensed
+        Material is used other than for NonCommercial purposes.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+-   a. Attribution.
+    -   1. If You Share the Licensed Material (including in modified
+        form), You must:
+        -   A. retain the following if it is supplied by the Licensor
+            with the Licensed Material:
+            -   i. identification of the creator(s) of the Licensed
+                Material and any others designated to receive
+                attribution, in any reasonable manner requested by the
+                Licensor (including by pseudonym if designated);
+            -   ii. a copyright notice;
+            -   iii. a notice that refers to this Public License;
+            -   iv. a notice that refers to the disclaimer of
+                warranties;
+            -   v. a URI or hyperlink to the Licensed Material to the
+                extent reasonably practicable;
+
+        -   B. indicate if You modified the Licensed Material and retain
+            an indication of any previous modifications; and
+        -   C. indicate the Licensed Material is licensed under this
+            Public License, and include the text of, or the URI or
+            hyperlink to, this Public License.
+    -   2. You may satisfy the conditions in Section 3(a)(1) in any
+        reasonable manner based on the medium, means, and context in
+        which You Share the Licensed Material. For example, it may be
+        reasonable to satisfy the conditions by providing a URI or
+        hyperlink to a resource that includes the required information.
+    -   3. If requested by the Licensor, You must remove any of the
+        information required by Section 3(a)(1)(A) to the extent
+        reasonably practicable.
+-   b. ShareAlike.In addition to the conditions in Section 3(a), if You
+    Share Adapted Material You produce, the following conditions also
+    apply.
+    -   1. The Adapter's License You apply must be a Creative Commons
+        license with the same License Elements, this version or later,
+        or a BY-NC-SA Compatible License.
+    -   2. You must include the text of, or the URI or hyperlink to, the
+        Adapter's License You apply. You may satisfy this condition in
+        any reasonable manner based on the medium, means, and context in
+        which You Share Adapted Material.
+    -   3. You may not offer or impose any additional or different terms
+        or conditions on, or apply any Effective Technological Measures
+        to, Adapted Material that restrict exercise of the rights
+        granted under the Adapter's License You apply.
+
+Section 4 – Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply
+to Your use of the Licensed Material:
+
+-   a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+    to extract, reuse, reproduce, and Share all or a substantial portion
+    of the contents of the database for NonCommercial purposes only;
+-   b. if You include all or a substantial portion of the database
+    contents in a database in which You have Sui Generis Database
+    Rights, then the database in which You have Sui Generis Database
+    Rights (but not its individual contents) is Adapted Material,
+    including for purposes of Section 3(b); and
+-   c. You must comply with the conditions in Section 3(a) if You Share
+    all or a substantial portion of the contents of the database.  
+    For the avoidance of doubt, this Section 4 supplements and does not
+    replace Your obligations under this Public License where the
+    Licensed Rights include other Copyright and Similar Rights.
+
+Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+-   a. Unless otherwise separately undertaken by the Licensor, to the
+    extent possible, the Licensor offers the Licensed Material as-is and
+    as-available, and makes no representations or warranties of any kind
+    concerning the Licensed Material, whether express, implied,
+    statutory, or other. This includes, without limitation, warranties
+    of title, merchantability, fitness for a particular purpose,
+    non-infringement, absence of latent or other defects, accuracy, or
+    the presence or absence of errors, whether or not known or
+    discoverable. Where disclaimers of warranties are not allowed in
+    full or in part, this disclaimer may not apply to You.
+-   b. To the extent possible, in no event will the Licensor be liable
+    to You on any legal theory (including, without limitation,
+    negligence) or otherwise for any direct, special, indirect,
+    incidental, consequential, punitive, exemplary, or other losses,
+    costs, expenses, or damages arising out of this Public License or
+    use of the Licensed Material, even if the Licensor has been advised
+    of the possibility of such losses, costs, expenses, or damages.
+    Where a limitation of liability is not allowed in full or in part,
+    this limitation may not apply to You.
+-   c. The disclaimer of warranties and limitation of liability provided
+    above shall be interpreted in a manner that, to the extent possible,
+    most closely approximates an absolute disclaimer and waiver of all
+    liability.
+
+Section 6 – Term and Termination.
+
+-   a. This Public License applies for the term of the Copyright and
+    Similar Rights licensed here. However, if You fail to comply with
+    this Public License, then Your rights under this Public License
+    terminate automatically.
+-   b. Where Your right to use the Licensed Material has terminated
+    under Section 6(a), it reinstates:
+
+    -   1. automatically as of the date the violation is cured, provided
+        it is cured within 30 days of Your discovery of the violation;
+        or
+    -   2. upon express reinstatement by the Licensor.
+
+    For the avoidance of doubt, this Section 6(b) does not affect any
+    right the Licensor may have to seek remedies for Your violations of
+    this Public License.
+
+-   c. For the avoidance of doubt, the Licensor may also offer the
+    Licensed Material under separate terms or conditions or stop
+    distributing the Licensed Material at any time; however, doing so
+    will not terminate this Public License.
+-   d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+    License.
+
+Section 7 – Other Terms and Conditions.
+
+-   a. The Licensor shall not be bound by any additional or different
+    terms or conditions communicated by You unless expressly agreed.
+-   b. Any arrangements, understandings, or agreements regarding the
+    Licensed Material not stated herein are separate from and
+    independent of the terms and conditions of this Public License.
+
+Section 8 – Interpretation.
+
+-   a. For the avoidance of doubt, this Public License does not, and
+    shall not be interpreted to, reduce, limit, restrict, or impose
+    conditions on any use of the Licensed Material that could lawfully
+    be made without permission under this Public License.
+-   b. To the extent possible, if any provision of this Public License
+    is deemed unenforceable, it shall be automatically reformed to the
+    minimum extent necessary to make it enforceable. If the provision
+    cannot be reformed, it shall be severed from this Public License
+    without affecting the enforceability of the remaining terms and
+    conditions.
+-   c. No term or condition of this Public License will be waived and no
+    failure to comply consented to unless expressly agreed to by the
+    Licensor.
+-   d. Nothing in this Public License constitutes or may be interpreted
+    as a limitation upon, or waiver of, any privileges and immunities
+    that apply to the Licensor or You, including from the legal
+    processes of any jurisdiction or authority.
+
+Creative Commons is not a party to its public licenses. Notwithstanding,
+Creative Commons may elect to apply one of its public licenses to
+material it publishes and in those instances will be considered the
+"Licensor." The text of the Creative Commons public licenses is
+dedicated to the public domain under the CC0 Public Domain Dedication.
+Except for the limited purpose of indicating that material is shared
+under a Creative Commons public license or as otherwise permitted by the
+Creative Commons policies published at creativecommons.org/policies,
+Creative Commons does not authorize the use of the trademark "Creative
+Commons" or any other trademark or logo of Creative Commons without its
+prior written consent including, without limitation, in connection with
+any unauthorized modifications to any of its public licenses or any
+other arrangements, understandings, or agreements concerning use of
+licensed material. For the avoidance of doubt, this paragraph does not
+form part of the public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.3
+Version: 0.0.10
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
+License: CC-BY-NC-SA-4.0
 Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
 Project-URL: Documentation, https://github.com/U4RASD/dalla-data-processing#readme
 Project-URL: Repository, https://github.com/U4RASD/dalla-data-processing
@@ -11,11 +12,15 @@ Keywords: arabic,nlp,data-processing,deduplication,stemming,readability,quality
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: <3.13,>=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: transformers>=4.30.0
 Requires-Dist: click>=8.0.0
@@ -42,6 +47,7 @@ Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
 Requires-Dist: pyyaml; extra == "pack"
 Provides-Extra: all
 Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"
+Dynamic: license-file
 
 # Dalla Data Processing (dalla-dp)
 

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/dalla_data_processing/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.3'
-__version_tuple__ = version_tuple = (0, 0, 3)
+__version__ = version = '0.0.10'
+__version_tuple__ = version_tuple = (0, 0, 10)
 
-__commit_id__ = commit_id = 'g37580acc9'
+__commit_id__ = commit_id = 'gcc87ead80'

{dalla_data_processing-0.0.3/dalla_data_processing/deduplication/onion/src_sc → dalla_data_processing-0.0.10/dalla_data_processing/deduplication/onion/src}/Makefile

@@ -1,7 +1,7 @@
 include ../Makefile.config
 
 CC=g++
-CFLAGS=-Wall -O3
+CFLAGS=-Wall -O3 -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/dalla_data_processing/deduplication/onion/src/Makefile.g

@@ -2,7 +2,7 @@ include ../Makefile.config
 
 CC=g++
 #CFLAGS=-Wall -O3
-CFLAGS=-Wall -g
+CFLAGS=-Wall -g -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.3/dalla_data_processing/deduplication/onion/src → dalla_data_processing-0.0.10/dalla_data_processing/deduplication/onion/src_sc}/Makefile

@@ -1,7 +1,7 @@
 include ../Makefile.config
 
 CC=g++
-CFLAGS=-Wall -O3
+CFLAGS=-Wall -O3 -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/dalla_data_processing/deduplication/onion/src_sc/Makefile.g

@@ -2,7 +2,7 @@ include ../Makefile.config
 
 CC=g++
 #CFLAGS=-Wall -O3
-CFLAGS=-Wall -g
+CFLAGS=-Wall -g -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/dalla_data_processing.egg-info/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.3
+Version: 0.0.10
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
+License: CC-BY-NC-SA-4.0
 Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
 Project-URL: Documentation, https://github.com/U4RASD/dalla-data-processing#readme
 Project-URL: Repository, https://github.com/U4RASD/dalla-data-processing
@@ -11,11 +12,15 @@ Keywords: arabic,nlp,data-processing,deduplication,stemming,readability,quality
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: <3.13,>=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: transformers>=4.30.0
 Requires-Dist: click>=8.0.0
@@ -42,6 +47,7 @@ Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
 Requires-Dist: pyyaml; extra == "pack"
 Provides-Extra: all
 Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"
+Dynamic: license-file
 
 # Dalla Data Processing (dalla-dp)
 

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/dalla_data_processing.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 .dockerignore
 .gitignore
 .pre-commit-config.yaml
+LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
@@ -14,6 +15,7 @@ dalla_data_processing.egg-info/PKG-INFO
 dalla_data_processing.egg-info/SOURCES.txt
 dalla_data_processing.egg-info/dependency_links.txt
 dalla_data_processing.egg-info/entry_points.txt
+dalla_data_processing.egg-info/not-zip-safe
 dalla_data_processing.egg-info/requires.txt
 dalla_data_processing.egg-info/top_level.txt
 dalla_data_processing/core/README.md

dalla_data_processing-0.0.10/dalla_data_processing.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=61.0", "setuptools-scm>=8.0", "wheel"]
+requires = ["setuptools>=70.0", "setuptools-scm>=8.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -11,13 +11,17 @@ authors = [
     {name = "Digital Research Unit - Arab Center", email = "dru@dohainstitute.edu.qa"}
 ]
 readme = "README.md"
-requires-python = ">=3.12,<3.13"
+license = {text = "CC-BY-NC-SA-4.0"}
+requires-python = ">=3.10"
 keywords = ["arabic", "nlp", "data-processing", "deduplication", "stemming", "readability", "quality"]
 classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Text Processing :: Linguistic",
 ]
@@ -76,6 +80,8 @@ Repository = "https://github.com/U4RASD/dalla-data-processing"
 packages = ["dalla_data_processing", "dalla_data_processing.core", "dalla_data_processing.deduplication", "dalla_data_processing.packing", "dalla_data_processing.stemming", "dalla_data_processing.quality", "dalla_data_processing.readability", "dalla_data_processing.utils"]
 include-package-data = true
 
+zip-safe = false
+
 [tool.setuptools.package-data]
 dalla_data_processing = ["py.typed"]
 "dalla_data_processing.stemming" = ["data/*.txt"]
@@ -83,7 +89,7 @@ dalla_data_processing = ["py.typed"]
 
 [tool.ruff]
 line-length = 100
-target-version = "py312"
+target-version = "py310"
 src = ["dalla_data_processing"]
 
 [tool.ruff.lint]

{dalla_data_processing-0.0.3 → dalla_data_processing-0.0.10}/scripts/build_onion.sh

@@ -59,18 +59,26 @@ make clean 2>/dev/null || true
 
 # Set up compiler flags for macOS
 EXTRA_CFLAGS=""
+EXTRA_LDFLAGS=""
 if [[ "$OSTYPE" == "darwin"* ]]; then
     # On macOS, add Homebrew paths for sparsehash
     if command -v brew &> /dev/null; then
         BREW_PREFIX=$(brew --prefix)
-        EXTRA_CFLAGS="-I${BREW_PREFIX}/include"
+        SPARSEHASH_PREFIX=$(brew --prefix google-sparsehash 2>/dev/null || echo "${BREW_PREFIX}")
+        EXTRA_CFLAGS="-I${SPARSEHASH_PREFIX}/include"
         echo -e "${YELLOW}Using Homebrew prefix: ${BREW_PREFIX}${NC}"
+        echo -e "${YELLOW}Using sparsehash include: ${SPARSEHASH_PREFIX}/include${NC}"
     fi
+    
+    # Build universal binary for macOS (x86_64 + arm64)
+    EXTRA_CFLAGS="${EXTRA_CFLAGS} -arch x86_64 -arch arm64"
+    EXTRA_LDFLAGS="-arch x86_64 -arch arm64"
+    echo -e "${YELLOW}Building universal binary (x86_64 + arm64)${NC}"
 fi
 
 # Build onion
 echo -e "${YELLOW}Compiling Onion...${NC}"
-if make CFLAGS="-Wall -O3 ${EXTRA_CFLAGS}"; then
+if make CFLAGS="-Wall -O3 -std=c++11 ${EXTRA_CFLAGS}" LDFLAGS="${EXTRA_LDFLAGS}"; then
     echo -e "${GREEN}✓ Compilation successful${NC}"
 else
     echo -e "${RED}✗ Compilation failed${NC}"
@@ -86,6 +94,12 @@ fi
 # Get platform info
 PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]')
 ARCH=$(uname -m)
+
+# On macOS, use "universal" as arch since we build a fat binary
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    ARCH="universal"
+fi
+
 echo -e "${YELLOW}Platform: $PLATFORM-$ARCH${NC}"
 
 # Create output directory
@@ -102,6 +116,12 @@ ln -sf "onion-$PLATFORM-$ARCH" "$OUTPUT_DIR/onion"
 echo -e "${GREEN}=== Build Complete ===${NC}"
 echo -e "${GREEN}Binary location: $OUTPUT_DIR/onion-$PLATFORM-$ARCH${NC}"
 
+# On macOS, verify it's a universal binary
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    echo -e "${YELLOW}Verifying universal binary architectures:${NC}"
+    lipo -info "$OUTPUT_DIR/onion-$PLATFORM-$ARCH"
+fi
+
 # Verify binary works
 if "$OUTPUT_DIR/onion-$PLATFORM-$ARCH" -h &> /dev/null; then
     echo -e "${GREEN}✓ Binary is executable and working${NC}"