PinInPy 1.0.2__tar.gz → 1.0.3__tar.gz

{pininpy-1.0.2 → pininpy-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PinInPy
-Version: 1.0.2
+Version: 1.0.3
 Summary: Python library for Pinyin searching
 Keywords: PinIn,pinyin match,pinyin,cffi
 Author-Email: SajoYukimi <kagiyamahino@gmail.com>
@@ -25,7 +25,7 @@ Classifier: Natural Language :: Chinese (Traditional)
 Project-URL: Homepage, https://github.com/KagiamamaHIna/PinInPy
 Project-URL: Bug Tracker, https://github.com/KagiamamaHIna/PinInPy/issues
 Project-URL: Source, https://github.com/KagiamamaHIna/PinInPy
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: cffi
 Description-Content-Type: text/markdown
 

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.12)
 include(GenerateExportHeader)
 
-project(PinIn4Cpp VERSION 1.5.3)
+project(PinIn4Cpp VERSION 1.5.7)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/include/PinIn4Cpp/PinIn.h

@@ -247,6 +247,7 @@ namespace PinInCpp {
 			PinIn& ctx;//直接绑定拼音上下文，方便reload
 			const std::string_view src;
 			std::vector<std::string_view> strs;//真正用于处理的数据
+			std::vector<detail::Utf8StringView> ustrs;
 		};
 		class Pinyin : public Element {
 		public:

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/include/PinIn4Cpp/TreeSearcher.h

@@ -116,7 +116,7 @@ namespace PinInCpp {
 	private:
 		void init() {
 			root = detail::SlabUniqueObj<Node>(NDensePool.NewObj());
-			acc.setProvider(&strs);
+			acc.SetProvider(&strs);
 			ticket = context->ticket([this]() {
 				for (const auto& i : this->naccs) {
 					i->reload(*this);
@@ -126,7 +126,7 @@ namespace PinInCpp {
 		}
 		void CommonSearch(std::string_view s, std::unordered_set<size_t>& ret) {
 			ticket->renew();
-			acc.search(s);
+			acc.SetSearchStr(s);
 			root->get(*this, ret, 0);
 		}
 		enum class NodeType : uint8_t {
@@ -145,7 +145,7 @@ namespace PinInCpp {
 			virtual void Serialize(std::vector<uint8_t>& data)const = 0;
 
 			//将自身载入对象池
-			virtual NodeType GetNodeType()const = 0;
+			virtual NodeType GetNodeType()const noexcept = 0;
 			static detail::SlabUniqueObj<Node> Deserialize(TreeSearcher& p, detail::VecU8Reader& reader);
 		};
 		//将Node*的所有权通过FreeToPool转移到对象池中，随后放弃其所有权并重设为新指针
@@ -180,7 +180,7 @@ namespace PinInCpp {
 			virtual Node* putRange(TreeSearcher& p, size_t start, size_t end);
 			virtual void Serialize(std::vector<uint8_t>& data)const;
 			static detail::SlabUniqueObj<NDense> Deserialize(TreeSearcher& p, detail::VecU8Reader& reader);
-			virtual NodeType GetNodeType()const {
+			virtual NodeType GetNodeType()const noexcept {
 				return NodeType::NDenseType;
 			}
 		private:
@@ -204,7 +204,7 @@ namespace PinInCpp {
 			virtual Node* putRange(TreeSearcher& p, size_t start, size_t end);
 			virtual void Serialize(std::vector<uint8_t>& data)const;
 			static detail::SlabUniqueObj<NSlice> Deserialize(TreeSearcher& p, detail::VecU8Reader& reader);
-			virtual NodeType GetNodeType()const {
+			virtual NodeType GetNodeType()const noexcept {
 				return NodeType::NSliceType;
 			}
 			NSlice(size_t start, size_t end) :start{ start }, end{ end } {}
@@ -229,7 +229,7 @@ namespace PinInCpp {
 			virtual void Serialize(std::vector<uint8_t>& data)const;
 			static detail::SlabUniqueObj<NMapTemplate> Deserialize(TreeSearcher& p, detail::VecU8Reader& reader);
 
-			virtual NodeType GetNodeType()const {
+			virtual NodeType GetNodeType()const noexcept {
 				return NodeType::NMapType;
 			}
 		private:
@@ -272,7 +272,7 @@ namespace PinInCpp {
 			void reload(TreeSearcher& p);
 			virtual void Serialize(std::vector<uint8_t>& data)const;
 			static detail::SlabUniqueObj<NAcc> Deserialize(TreeSearcher& p, detail::VecU8Reader& reader);
-			virtual NodeType GetNodeType()const {
+			virtual NodeType GetNodeType()const noexcept {
 				return NodeType::NAccType;
 			}
 		private:
@@ -296,6 +296,9 @@ namespace PinInCpp {
 		constexpr static size_t NDenseThreshold = 128;
 		//表节点转换临界点
 		constexpr static size_t NMapThreshold = 32;
+		//预分配多少元素给搜索的结果集，这将影响搜索性能
+		//	由于通用内存分配器很慢，而size_t足够廉价，调大点对内存影响不是很大但是对大的搜索集的构建来讲会快
+		constexpr static size_t ResultSetPreSize = 10000;
 		const Logic logic;
 		std::shared_ptr<PinIn> context = nullptr;//PinIn
 		detail::Accelerator acc;
@@ -310,7 +313,7 @@ namespace PinInCpp {
 	/* NMapTemplate 实现(避免前面的声明过长) */
 	template<bool CanUpgrade>
 	void TreeSearcher::NMapTemplate<CanUpgrade>::get(TreeSearcher& p, std::unordered_set<size_t>& ret, size_t offset) {
-		if (p.acc.search().size() == offset) {
+		if (p.acc.GetSearchStr().size() == offset) {
 			if (p.logic == Logic::EQUAL) {
 				leaves.AddToSTLSet(ret);
 			}

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/include/PinIn4Cpp/detail/Accelerator.h

@@ -8,18 +8,19 @@ namespace PinInCpp {
 		public:
 			Accelerator(PinIn& p) : ctx{ p } {
 			}
-			const Utf8StringView& search() {
+			const Utf8StringView& GetSearchStr() {
 				return searchStr;
 			}
-			uint32_t searchU32FourCC(size_t i) {
+			uint32_t GetSearchU32FourCC(size_t i) {
 				return u32strVec[i];
 			}
-			void search(std::string_view s) {
+			void SetSearchStr(std::string_view s) {
 				if (s != searchSrcStr) {
 					searchSrcStr = s;
 					searchStr.reset(searchSrcStr);
 					if (cache.size() < searchStr.size()) {
 						cache.resize(searchStr.size());
+						pinyinCache.resize(searchStr.size());
 					}
 
 					u32strVec.clear();
@@ -33,9 +34,12 @@ namespace PinInCpp {
 				for (auto& v : cache) {
 					v.clear();//仅清空map容器，避免map对象再构造
 				}
+				for (auto& v : pinyinCache) {
+					v.clear();
+				}
 			}
 			//接收一个外部的、长生命周期的provider，不拥有
-			void setProvider(UTF8StringPool* provider_ptr) {
+			void SetProvider(UTF8StringPool* provider_ptr) {
 				provider = provider_ptr;
 			}
 			void ShrinkToFit() {
@@ -45,7 +49,6 @@ namespace PinInCpp {
 				cache.shrink_to_fit();
 			}
 
-			IndexSet get(const PinIn::Pinyin& p, size_t offset);
 			IndexSet get(const uint32_t ch, size_t offset);
 			size_t common(size_t s1, size_t s2, size_t max);
 			bool check(size_t offset, size_t start);
@@ -56,10 +59,12 @@ namespace PinInCpp {
 				return searchStr;
 			}
 		private:
+			IndexSet get(const PinIn::Pinyin& p, size_t offset);
 			UTF8StringPool* provider = nullptr;     //观察者指针，不拥有
 
 			PinIn& ctx;
-			std::vector<IndexSet::Storage> cache;
+			std::vector<std::unordered_map<uint32_t, IndexSet>> cache;
+			std::vector<IndexSet::Storage> pinyinCache;
 			Utf8StringView searchStr;
 			std::string searchSrcStr;
 			std::vector<uint32_t> u32strVec;

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/include/PinIn4Cpp/detail/AdaptiveContainers.h

@@ -242,7 +242,7 @@ namespace PinInCpp {
 			//AdaptiveSet转换临界点
 			constexpr static size_t ContainerThreshold = 128;
 
-			static constexpr uint8_t SVONum = sizeof(std::vector<value>) / sizeof(value);//计算一个合适的长度
+			static constexpr uint8_t SVONum = (sizeof(std::vector<value>) - sizeof(size_t)) / sizeof(value);//计算一个合适的长度
 			static constexpr bool isSVO = SVONum > 0;//计算是否可以无损SVO优化
 			using ObjVector = std::conditional_t<isSVO, SVOArray<value, SVONum>, std::vector<value>>;
 
@@ -405,7 +405,7 @@ namespace PinInCpp {
 			using KVPair = std::pair<const k, v>;
 			//AdaptiveSet转换临界点
 			constexpr static size_t ContainerThreshold = 128;
-			static constexpr uint8_t SVONum = sizeof(std::vector<KVPair>) / sizeof(KVPair);//计算一个合适的长度
+			static constexpr uint8_t SVONum = (sizeof(std::vector<KVPair>) - sizeof(size_t)) / sizeof(KVPair);//计算一个合适的长度
 			static constexpr bool isSVO = SVONum > 0;//计算是否可以无损SVO优化
 			using ObjVector = std::conditional_t<isSVO, SVOArray<KVPair, SVONum>, std::vector<KVPair>>;
 

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/src/Keyboard.cpp

@@ -240,16 +240,6 @@ namespace PinInCpp {
 	}
 
 	std::vector<std::string_view> Keyboard::Zero(std::string_view s) {
-		std::vector<std::string_view> ss = Standard(s);
-		if (ss.size() == 1) {//因为职责改变，所以是1，没有声调
-			std::string_view finale = ss[0];//取字符串第一个元素
-			ss[0] = finale.substr(0, 1);//覆写第一个元素为其字符串开头的字符
-			ss.emplace_back(finale);//因为职责改变，去除了声调在这里，所以只有一个音素的情况下，直接最后追加即可
-		}
-		return ss;
-	}
-
-	std::vector<std::string_view> Keyboard::ZeroZiranmaOrXiaohe(std::string_view s) {
 		std::vector<std::string_view> ss = Standard(s);
 		if (ss.size() == 1) {//因为职责改变，所以是1，没有声调
 			std::string_view finale = ss[0];//取字符串第一个元素
@@ -354,7 +344,7 @@ namespace PinInCpp {
 		{"ing", "j"}, {"iong", "s"}, {"iu", "y"}, {"ong", "s"}, {"ou", "p"},
 		{"sh", "u"}, {"ua", "q"}, {"uai", "y"}, {"uan", "w"}, {"uang", "n"},
 		{"ui", "v"}, {"un", "z"}, {"uo", "o"}, {"van", "w"}, {"ve", "x"},
-		{"vn", "z"}, {"zh", "v"},
+		{"ue", "x"}, {"vn", "z"}, {"zh", "v"},
 		});
 
 	const static std::map<std::string, std::string> MICROSOFT_KEYS = std::map<std::string, std::string>({
@@ -387,8 +377,8 @@ namespace PinInCpp {
 
 	const Keyboard Keyboard::QUANPIN = Keyboard(std::nullopt, std::nullopt, Keyboard::Standard, false, true);
 	const Keyboard Keyboard::DAQIAN = Keyboard(PHONETIC_LOCAL, DAQIAN_KEYS, Keyboard::Standard, false, false);
-	const Keyboard Keyboard::XIAOHE = Keyboard(std::nullopt, XIAOHE_KEYS, Keyboard::ZeroZiranmaOrXiaohe, true, false);
-	const Keyboard Keyboard::ZIRANMA = Keyboard(std::nullopt, ZIRANMA_KEYS, Keyboard::ZeroZiranmaOrXiaohe, true, false);
+	const Keyboard Keyboard::XIAOHE = Keyboard(std::nullopt, XIAOHE_KEYS, Keyboard::Zero, true, false);
+	const Keyboard Keyboard::ZIRANMA = Keyboard(std::nullopt, ZIRANMA_KEYS, Keyboard::Zero, true, false);
 	const Keyboard Keyboard::SOUGOU = Keyboard(std::nullopt, SOUGOU_KEYS, Keyboard::ZeroOInitial, true, false);
 	const Keyboard Keyboard::ZHINENG_ABC = Keyboard(std::nullopt, ZHINENG_ABC_KEYS, Keyboard::ZeroOInitial, true, false);
 	const Keyboard Keyboard::GUOBIAO = Keyboard(std::nullopt, GUOBIAO_KEYS, Keyboard::ZeroAInitial, true, false);

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/src/PinIn.cpp

@@ -476,7 +476,7 @@ namespace PinInCpp {
 		if (empty()) {
 			return result;
 		}
-		for (const auto& str : strs) {
+		for (const auto& str : ustrs) {
 			size_t size = StrCmp(source, str, start);
 			if (size == 0) {
 				continue;
@@ -589,11 +589,13 @@ namespace PinInCpp {
 
 	void PinIn::Phoneme::reload() {
 		strs.clear();//应该前置，因为是在重载，非法的话就当然置空了
+		ustrs.clear();
 		if (src.empty()) {//没数据？非法的吧！，不过就直接结束了也算一种处理了
 			return;
 		}
 		if (src.size() == 1 && src[0] >= '0' && src[0] <= '4') {
 			strs.emplace_back(src); //声调就是它自己，直接处理完毕返回！
+			ustrs.emplace_back(src);
 			return;
 		}
 		if (ctx.keyboard.GetHasFuuzyLocal()) {
@@ -602,6 +604,9 @@ namespace PinInCpp {
 		else {
 			reloadNoMap();//标准音素，纯逻辑实现
 		}
+		for (const auto& v : strs) {
+			ustrs.emplace_back(v);
+		}
 	}
 
 	void PinIn::Pinyin::reload(std::string_view str) {

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/src/TreeSearcher.cpp

@@ -14,6 +14,7 @@ namespace PinInCpp {
 
 	std::vector<std::string> TreeSearcher::ExecuteSearch(std::string_view s) {
 		std::unordered_set<size_t> ret;
+		ret.reserve(ResultSetPreSize);
 		CommonSearch(s, ret);
 
 		std::vector<std::string> result;
@@ -26,6 +27,7 @@ namespace PinInCpp {
 
 	std::unordered_set<size_t> TreeSearcher::ExecuteSearchGetSet(std::string_view s) {
 		std::unordered_set<size_t> ret;
+		ret.reserve(ResultSetPreSize);
 		CommonSearch(s, ret);
 		return ret;
 	}
@@ -58,7 +60,7 @@ namespace PinInCpp {
 
 	void TreeSearcher::NDense::get(TreeSearcher& p, std::unordered_set<size_t>& ret, size_t offset) {
 		bool full = p.logic == Logic::EQUAL;
-		if (!full && p.acc.search().size() == offset) {
+		if (!full && p.acc.GetSearchStr().size() == offset) {
 			get(p, ret);
 		}
 		else {
@@ -165,7 +167,7 @@ namespace PinInCpp {
 	}
 
 	void TreeSearcher::NAcc::get(TreeSearcher& p, std::unordered_set<size_t>& result, size_t offset) {
-		if (p.acc.search().size() == offset) {
+		if (p.acc.GetSearchStr().size() == offset) {
 			if (p.logic == Logic::EQUAL) {
 				NodeMap.leaves.AddToSTLSet(result);
 			}
@@ -174,12 +176,12 @@ namespace PinInCpp {
 			}
 		}
 		else {
-			auto it = NodeMap.children->find(p.acc.searchU32FourCC(offset));
+			auto it = NodeMap.children->find(p.acc.GetSearchU32FourCC(offset));
 			if (it != nullptr) {
 				it->get()->get(p, result, offset + 1);
 			}
 			for (const auto& [k, v] : index_node) {
-				if (!k->match(p.acc.search(), offset, true).empty()) {
+				if (!k->match(p.acc.GetSearchStr(), offset, true).empty()) {
 					detail::AdaptiveMap<uint32_t, detail::SlabUniqueObj<Node>>& map = *NodeMap.children;
 					v.forEach([&](uint32_t c) {
 						detail::IndexSet::IndexSetIterObj it = p.acc.get(c, offset).GetIterObj();
@@ -343,7 +345,7 @@ namespace PinInCpp {
 		if (this->start + start == end) {
 			exit_node->get(p, ret, offset);
 		}
-		else if (offset == p.acc.search().size()) {
+		else if (offset == p.acc.GetSearchStr().size()) {
 			if (p.logic != Logic::EQUAL) {
 				exit_node->get(p, ret);
 			}

{pininpy-1.0.2 → pininpy-1.0.3}/PinIn4Cpp/src/utils/Accelerator.cpp

@@ -3,7 +3,7 @@
 namespace PinInCpp {
 	namespace detail {
 		IndexSet Accelerator::get(const PinIn::Pinyin& p, size_t offset) {
-			IndexSet::Storage& data = cache[offset];
+			IndexSet::Storage& data = pinyinCache[offset];
 			IndexSet ret = data.get(p.id);
 			if (ret == IndexSet::NONE) {
 				ret = p.match(searchStr, offset, partial);
@@ -14,6 +14,10 @@ namespace PinInCpp {
 		}
 
 		IndexSet Accelerator::get(const uint32_t ch, size_t offset) {
+			auto it = cache[offset].find(ch);
+			if (it != cache[offset].end()) {
+				return it->second;
+			}
 			PinIn::Character* c = ctx.GetCharCachePtr(ch);
 			if (c == nullptr) {
 				PinIn::Character c = ctx.GetChar(ch);
@@ -21,6 +25,7 @@ namespace PinInCpp {
 				for (const PinIn::Pinyin* p : c.GetPinyins()) {
 					ret.merge(get(*p, offset));
 				}
+				cache[offset].insert_or_assign(ch, ret);
 				return ret;
 			}
 			else {
@@ -28,6 +33,7 @@ namespace PinInCpp {
 				for (const PinIn::Pinyin* p : c->GetPinyins()) {
 					ret.merge(get(*p, offset));
 				}
+				cache[offset].insert_or_assign(ch, ret);
 				return ret;
 			}
 		}

{pininpy-1.0.2 → pininpy-1.0.3}/pyproject.toml

@@ -4,9 +4,9 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "PinInPy"
-version = "1.0.2"
+version = "1.0.3"
 dependencies = ["cffi"]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 authors = [
     { name = "SajoYukimi", email = "kagiyamahino@gmail.com" },
 ]