removedup 1.0.2__tar.gz → 1.0.4__tar.gz

{removedup-1.0.2/removedup.egg-info → removedup-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: removedup
-Version: 1.0.2
+Version: 1.0.4
 Summary: Remove duplicates from parallel corpora
 Home-page: https://github.com/LibreTranslate/RemoveDUP
 Author: Piero Toffanin

removedup-1.0.4/dedup.cpp

@@ -0,0 +1,45 @@
+#include "dedup.hpp"
+
+std::tuple<std::string, std::string, size_t> dedup(const std::string &src, const std::string &tgt){
+    std::ifstream src_if(src, std::ios_base::in | std::ios_base::binary);
+    if (!src_if.is_open()) throw std::runtime_error("Cannot open " + src);
+    
+    std::ifstream tgt_if(tgt, std::ios_base::in | std::ios_base::binary);
+    if (!tgt_if.is_open()) throw std::runtime_error("Cannot open " + tgt);
+    
+    std::unordered_map<std::uint32_t, bool> seen;
+
+    std::string src_out = src + ".dedup";
+    std::string tgt_out = tgt + ".dedup";
+
+    std::ofstream src_of(src_out, std::ios::trunc);
+    std::ofstream tgt_of(tgt_out, std::ios::trunc);
+
+    std::string line_s = "";
+    std::string line_t = "";
+    size_t removed = 0;
+
+    while(true) {
+        std::getline(src_if, line_s);
+        std::getline(tgt_if, line_t);
+        if (src_if.eof()) break;
+
+        std::uint32_t hash = xxh32::hash(line_s.c_str(), static_cast<uint32_t>(line_s.size()), 0);
+
+        bool line_exists = seen.count(hash) > 0;
+        if (!line_exists) {
+            seen[hash] = true;
+            src_of << line_s << "\n";
+            tgt_of << line_t << "\n";
+        }else{
+            removed++;
+        }
+    }
+
+    src_of.close();
+    tgt_of.close();
+    src_if.close();
+    tgt_if.close();
+
+    return std::make_tuple(src_out, tgt_out, removed);
+}

removedup-1.0.4/dedup.hpp

@@ -0,0 +1,10 @@
+#include <iostream>
+#include <fstream>
+#include <unordered_map>
+#include <cstring>
+#include <tuple>
+#include <string>
+
+#include "vendor/xxh32.hpp"
+
+std::tuple<std::string, std::string, size_t> dedup(const std::string &src, const std::string &tgt);

{removedup-1.0.2 → removedup-1.0.4/removedup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: removedup
-Version: 1.0.2
+Version: 1.0.4
 Summary: Remove duplicates from parallel corpora
 Home-page: https://github.com/LibreTranslate/RemoveDUP
 Author: Piero Toffanin

{removedup-1.0.2 → removedup-1.0.4}/setup.py

@@ -124,7 +124,7 @@ class CMakeBuild(build_ext):
 
 setup(
     name="removedup",
-    version="1.0.2",
+    version="1.0.4",
     author="Piero Toffanin",
     author_email="pt@masseranolabs.com",
     url="https://github.com/LibreTranslate/RemoveDUP",

removedup-1.0.2/dedup.cpp

@@ -1,102 +0,0 @@
-#include "dedup.hpp"
-
-std::tuple<std::string, std::string, size_t> dedup(const std::string &src, const std::string &tgt){
-    #if defined(_WIN32) || defined(_WIN64)
-    HANDLE file_handle = CreateFile(src.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-    if (file_handle == INVALID_HANDLE_VALUE) {
-        throw std::runtime_error("Unable to open " + src);
-    }
-
-    DWORD file_size = GetFileSize(file_handle, NULL);
-    HANDLE file_mapping = CreateFileMapping(file_handle, NULL, PAGE_READONLY, 0, 0, NULL);
-    if (file_mapping == NULL) {
-        CloseHandle(file_handle);
-        throw std::runtime_error("File mapping failed.");
-    }
-
-    const char* file_data = static_cast<const char*>(MapViewOfFile(file_mapping, FILE_MAP_READ, 0, 0, file_size));
-    if (file_data == NULL) {
-        CloseHandle(file_mapping);
-        CloseHandle(file_handle);
-        throw std::runtime_error("Memory mapping failed.");
-    }
-#else
-    int fd = open(src.c_str(), O_RDONLY);
-    if (fd == -1) throw std::runtime_error("Unable to open " + src);
-
-    struct stat file_stat;
-    if (fstat(fd, &file_stat) == -1) {
-        close(fd);
-        throw std::runtime_error("Unable to get file size for " + src);
-    }
-
-    off_t file_size = file_stat.st_size;
-    const char* file_data = static_cast<const char*>(mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0));
-    if (file_data == MAP_FAILED) {
-        close(fd);
-        throw std::runtime_error("Memory mapping failed.");
-    }
-
-#endif
-
-    std::unordered_multimap<std::uint32_t, std::string_view> lines;
-
-    std::ifstream tgt_is(tgt);
-    if (!tgt_is.is_open()) throw std::runtime_error("Cannot open " + tgt);
-
-    std::string src_out = src + ".dedup";
-    std::string tgt_out = tgt + ".dedup";
-
-    std::ofstream src_of(src_out, std::ios::trunc);
-    std::ofstream tgt_of(tgt_out, std::ios::trunc);
-
-    const char* line_start = file_data;
-    const char* line_end = file_data;
-    std::string line_buf = "";
-    size_t removed = 0;
-
-    for (const char* ptr = file_data; ptr < file_data + file_size; ++ptr) {
-        if (*ptr == '\n' || ptr == file_data + file_size - 1) {
-            std::getline(tgt_is, line_buf);
-
-            line_end = ptr + 1;
-            std::string_view line_view(line_start, line_end - line_start);
-
-            std::uint32_t hash = xxh32::hash(line_start, static_cast<uint32_t>(line_end - line_start), 0);
-            auto range = lines.equal_range(hash);
-
-            bool line_exists = false;
-            for (auto it = range.first; it != range.second; ++it) {
-                if (it->second == line_view) {
-                    line_exists = true;
-                    break;
-                }
-            }
-
-            if (!line_exists) {
-                lines.emplace(hash, line_view);
-                
-                src_of << line_view;
-                tgt_of << line_buf << (*ptr == '\n' ? "\n" : "");
-            }else{
-                removed++;
-            }
-
-            line_start = line_end;
-        }
-    }
-
-#if defined(_WIN32) || defined(_WIN64)
-    UnmapViewOfFile(file_data);
-    CloseHandle(file_mapping);
-    CloseHandle(file_handle);
-#else
-    munmap(const_cast<char*>(file_data), file_stat.st_size);
-    close(fd);
-#endif
-
-    src_of.close();
-    tgt_of.close();
-
-    return std::make_tuple(src_out, tgt_out, removed);
-}

removedup-1.0.2/dedup.hpp

@@ -1,21 +0,0 @@
-#include <iostream>
-#include <fstream>
-#include <unordered_map>
-#include <string_view>
-#include <cstring>
-#include <tuple>
-#include <variant>
-#include <string>
-
-#if defined(_WIN32) || defined(_WIN64)
-#include <windows.h>
-#else
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#endif
-
-#include "vendor/xxh32.hpp"
-
-std::tuple<std::string, std::string, size_t> dedup(const std::string &src, const std::string &tgt);