first commit

Rime/tools/clear.py

@@ -0,0 +1,25 @@
+import os, sys, re
+
+workDir = os.path.abspath(os.path.dirname(sys.path[0]))
+pattern = re.compile(r"^[\u4e00-\u9fa5]+$")
+
+with open(
+    os.path.join(workDir, "opencc", "emoji_word.txt"), "r", encoding="utf-8"
+) as f:
+    text = f.read()
+
+lines = text.split("\n")
+
+with open(
+    os.path.join(workDir, "opencc", "emoji_word.txt"),
+    "w",
+    encoding="utf-8",
+    newline="\n",
+) as outFile:
+    for line in lines:
+        if line:
+            parts = line.split("\t")
+            if pattern.match(parts[0]):
+                outFile.write(line + "\n")
+
+print("Clear non-Chinese characters successfully")

Rime/tools/emoji2dict.py

@@ -0,0 +1,46 @@
+import os, sys, re, datetime
+from pypinyin import pinyin, Style
+
+workDir = os.path.abspath(os.path.dirname(sys.path[0]))
+pattern = re.compile(r"^[\u4e00-\u9fa5]+$")
+
+# now = datetime.datetime.now().strftime("%Y-%m-%d")
+# dateStr = now.strftime("%Y-%m-%d")
+infoStr = f"""# Rime dictionary
+# encoding: utf-8
+# source: https://github.com/iDvel/rime-ice/blob/main/opencc/emoji.txt
+
+---
+name: emoji
+version: "{datetime.datetime.now().strftime("%Y-%m-%d")}"
+sort: by_weight
+...
+
+"""
+
+with open(
+    os.path.join(workDir, "opencc", "emoji_word.txt"), "r", encoding="utf-8"
+) as f:
+    text = f.read()
+
+lines = text.split("\n")
+
+
+with open(
+    os.path.join(workDir, "dicts", "emoji.dict.yaml"),
+    "w",
+    encoding="utf-8",
+    newline="\n",
+) as outFile:
+    outFile.write(infoStr)
+    for line in lines:
+        if line:
+            parts = line.split("\t")
+            # emoji = parts[1].strip().split(" ")[-1]
+            name = parts[0].strip().split(" ")[-1]
+            pinyin_name = " ".join([p[0] for p in pinyin(name, style=Style.NORMAL)])
+            if pattern.match(name):
+                # print(f"{name}\t{pinyin_name}")
+                outFile.write(f"{name}\t{pinyin_name}\n")
+
+print("Emoji to dictionary successfully")

Rime/tools/recipes/enable_ascii_composer.recipe.yaml

@@ -0,0 +1,20 @@
+# encoding: utf-8
+---
+recipe:
+  Rx: tools/recipes/enable_ascii_composer
+  description: >-
+    Enable ascii composer for all schema
+patch:
+  engine/processors/@before 0: ascii_composer
+
+patch_files:
+  lufs_dpy.schema.yaml:
+    - engine/processors/@before 0: ascii_composer
+  lufs_flypy.schema.yaml:
+    - engine/processors/@before 0: ascii_composer
+  lufs_mspy.schema.yaml:
+    - engine/processors/@before 0: ascii_composer
+  lufs_pinyin.schema.yaml:
+    - engine/processors/@before 0: ascii_composer
+  lufs_pyjj.schema.yaml:
+    - engine/processors/@before 0: ascii_composer

Rime/tools/recipes/install.recipe.yaml

@@ -0,0 +1,19 @@
+# encoding: utf-8
+---
+recipe:
+  Rx: tools/recipes/install
+  description: >-
+    Full install LufsX/rime
+install_files: >-
+  custom_phrase.txt
+  default.yaml
+  dicts/*.*
+  extended.dict.yaml
+  lua/*.*
+  lua/*/*.*
+  lufs*.yaml
+  opencc/*.*
+  rime.lua
+  squirrel.yaml
+  weasel.yaml
+  zh-hans-t-essay-bgw.gram

Rime/tools/recipes/update.recipe.yaml

@@ -0,0 +1,9 @@
+# encoding: utf-8
+---
+recipe:
+  Rx: tools/recipes/update
+  description: >-
+    Update dicts and opencc only
+install_files: >-
+  dicts/*.*
+  opencc/*.*

Rime/tools/run.py

@@ -0,0 +1,8 @@
+import os,sys
+
+workDir = os.path.abspath(os.path.dirname(sys.path[0]))
+
+list=["update.py","clear.py","emoji2dict.py","util.py"]
+
+for file in list:
+    os.system(f"python {os.path.join(workDir,"tools",file)}")

Rime/tools/update.py

@@ -0,0 +1,23 @@
+import requests, os, sys
+
+workDir = os.path.abspath(os.path.dirname(sys.path[0]))
+
+update_url = {
+    os.path.join(
+        workDir, "dicts", "simp.dict.yaml"
+    ): "https://github.com/iDvel/rime-ice/raw/main/cn_dicts/8105.dict.yaml",
+    os.path.join(
+        workDir, "opencc", "emoji_word.txt"
+    ): "https://github.com/iDvel/rime-ice/raw/main/opencc/emoji.txt",
+}
+
+for path, link in update_url.items():
+    try:
+        response = requests.get(link)
+        if response.status_code == 200:
+            with open(path, "wb") as f:
+                f.write(response.content)
+    except Exception as e:
+        print(f"{path} 报错: {str(e)}")
+
+print("Update successful")

Rime/tools/util.py

@@ -0,0 +1,135 @@
+def deduplicate(src_file, dest_file):
+    lines_seen = set()
+    output_lines = []
+
+    with open(src_file, "r", encoding="utf-8") as file:
+        for line in file:
+            stripped_line = line.strip()
+            if (
+                stripped_line == ""
+                or stripped_line.startswith("#")
+                or stripped_line not in lines_seen
+            ):
+                output_lines.append(line)
+                if stripped_line != "":
+                    lines_seen.add(stripped_line)
+
+    with open(dest_file, "w", encoding="utf-8", newline="\n") as file:
+        file.writelines(output_lines)
+
+
+def get_info(src_file):
+    with open(src_file, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    info = []
+    flag_continue = False
+
+    for line in lines:
+        if line == "\n" or line.startswith("#"):
+            info.append(line)
+        elif line == "---\n":
+            info.append(line)
+            flag_continue = True
+        elif flag_continue:
+            info.append(line)
+            if line == "...\n":
+                flag_continue = False
+        else:
+            break
+
+    return info
+
+
+def sort(src_file, dest_file):
+    info = get_info(src_file)
+
+    # 跳过处理 THUOCL 词库
+    if "thuocl" in src_file:
+        return
+
+    with open(src_file, "r", encoding="utf-8") as f:
+        lines = f.readlines()[len(info) :]
+
+    format_type = len(lines[0].split("\t"))
+
+    # format_type == 2 时，排序「字	zi」此类，规则是按拼音排序
+    # format_type == 3 时，排序「字	zi	1」此类，规则是按拼音+字频排序
+    # 暂时不支持混排
+    if format_type == 2:
+        wait_to_write = []
+        part_to_sort = []
+
+        for line in lines:
+            if line.startswith("#") or line == "\n":
+                if part_to_sort:
+                    sorted_lines = sorted(
+                        part_to_sort,
+                        key=lambda x: x.split("\t")[1]
+                        if len(x.split("\t")) > 1
+                        else "",
+                    )
+                    wait_to_write += sorted_lines
+                    part_to_sort.clear()
+                wait_to_write.append(line)
+                continue
+            part_to_sort.append(line)
+
+        if part_to_sort:
+            wait_to_write += sorted(
+                part_to_sort,
+                key=lambda x: x.split("\t")[1] if len(x.split("\t")) > 1 else "",
+            )
+        with open(dest_file, "w", encoding="utf-8", newline="\n") as f:
+            f.writelines(info + wait_to_write)
+    elif format_type == 3:
+        wait_to_write = []
+        part_to_sort = []
+
+        for line in lines:
+            if line.startswith("#") or line == "\n":
+                if part_to_sort:
+                    sorted_lines = sorted(
+                        part_to_sort,
+                        key=lambda x: (
+                            x.split("\t")[1] if len(x.split("\t")) > 1 else "",
+                            int(x.split("\t")[2])
+                            if len(x.split("\t")) > 2 and x.split("\t")[2].isdigit()
+                            else 0,
+                        ),
+                    )
+                    wait_to_write += sorted_lines
+                    part_to_sort.clear()
+                wait_to_write.append(line)
+                continue
+            part_to_sort.append(line)
+
+        if part_to_sort:
+            wait_to_write += sorted(
+                part_to_sort,
+                key=lambda x: (
+                    x.split("\t")[1] if len(x.split("\t")) > 1 else "",
+                    int(x.split("\t")[2])
+                    if len(x.split("\t")) > 2 and x.split("\t")[2].isdigit()
+                    else 0,
+                ),
+            )
+        with open(dest_file, "w", encoding="utf-8", newline="\n") as f:
+            f.writelines(info + wait_to_write)
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+
+    dicts_dir = os.path.join(os.path.abspath(os.path.dirname(sys.path[0])), "dicts")
+
+    for root, _, files in os.walk(dicts_dir):
+        for file in files:
+            if file.endswith(".yaml"):
+                file_path = os.path.join(root, file)
+                print(f"Processing {file_path}")
+                deduplicate(file_path, file_path)
+                sort(file_path, file_path)
+
+    print("Processing completed")