10-12-2025, 08:34 AM
I conducted a small experiment with LAAFU. The initial question is: In the absence of vowels and consonants, is it possible to form chunks from Voynich words based purely on statistical frequency and then use them for further analysis? Here is the result.
Here are the twelve most frequent chunks (start/middle/end), each listed
Frequent pairs (start chunk first word of the line, end chunk last word of the line)
RF1a-n-x7_line_start_end_pairs.xlsx (Size: 29.65 KB / Downloads: 8)
Code:
from collections import Counter
# -----------------------------
# Settings / Configuration
# -----------------------------
INPUT_FILE = "RF1a-n-x7.txt"
MIN_COUNT = 2
MIN_LEN = 2
MAX_LEN = 10
OUTPUT_FILE_CHUNKS = "RF1a-n-x7_chunked.txt"
OUTPUT_FILE_TOP_CHUNKS = "RF1a-n-x7_top_chunks.txt"
OUTPUT_FILE_LINES = "RF1a-n-x7_line_start_end_pairs.txt"
OUTPUT_FILE_LINES_CSV = "RF1a-n-x7_line_start_end_pairs.csv"
# -----------------------------
# 1) Read all words line by line
# -----------------------------
words = []
lines = []
with open(INPUT_FILE, "r", encoding="utf-8") as f:
for line in f:
line_words = line.strip().split()
if line_words:
lines.append(line_words)
words.extend(line_words)
# -----------------------------
# 2) Count all substrings (potential chunks)
# -----------------------------
freq = Counter()
for w in words:
L = len(w)
for l in range(MIN_LEN, min(MAX_LEN, L) + 1):
for i in range(L - l + 1):
substring = w[i:i+l]
freq[substring] += 1
# Keep only substrings that occur at least MIN_COUNT times
common = {s for s, c in freq.items() if c >= MIN_COUNT}
# -----------------------------
# 3) Function: Chunk a single word
# -----------------------------
def chunk_word(word):
"""
Break a word into statistically significant chunks.
Chunks are scored by: frequency * length
Highest score wins.
"""
chunks = []
i = 0
L = len(word)
while i < L:
candidates = []
for l in range(MIN_LEN, min(MAX_LEN, L - i) + 1):
piece = word[i:i+l]
if piece in common:
score = freq[piece] * l
candidates.append((score, l, piece))
if candidates:
candidates.sort(key=lambda x: (-x[0], -x[1]))
_, l, match = candidates[0]
chunks.append(match)
i += l
else:
chunks.append(word[i])
i += 1
return chunks
# -----------------------------
# 4) Process each line and collect positional statistics
# -----------------------------
word_start = Counter() # first chunk in word
word_mid = Counter() # middle chunks
word_end = Counter() # last chunk in word
line_pairs = Counter() # Start → End pairs per line
chunked_words = []
for line_words in lines:
start_chunk_line = None
end_chunk_line = None
for idx, w in enumerate(line_words):
chunks = chunk_word(w)
chunked_words.append(f"{w}: {','.join(chunks)}")
if chunks:
# register word-start/mid/end per word
word_start[chunks[0]] += 1
word_end[chunks[-1]] += 1
for c in chunks[1:-1]:
word_mid[c] += 1
# For the line-level start-end pair:
if idx == 0 and chunks:
start_chunk_line = chunks[0]
if idx == len(line_words) - 1 and chunks:
end_chunk_line = chunks[-1]
if start_chunk_line and end_chunk_line:
line_pairs[(start_chunk_line, end_chunk_line)] += 1
# -----------------------------
# 5) Save chunked words to file
# -----------------------------
with open(OUTPUT_FILE_CHUNKS, "w", encoding="utf-8") as f:
for line in chunked_words:
f.write(line + "\n")
# -----------------------------
# 6) Function: Print top chunks
# -----------------------------
def print_top_chunks(counter, title, top_n=12):
total = sum(counter.values())
print(f"\n{title}")
print("-" * 40)
for chunk, count in counter.most_common(top_n):
pct = count / total * 100
print(f"{chunk:<6} {count:6} ({pct:5.2f}%)")
# -----------------------------
# 7) Save top chunks to file
# -----------------------------
with open(OUTPUT_FILE_TOP_CHUNKS, "w", encoding="utf-8") as f:
for counter, title in [(word_start, "Word-Start Chunks"),
(word_mid, "Word-Mid Chunks"),
(word_end, "Word-End Chunks")]:
f.write(f"{title}\n")
f.write("-" * 40 + "\n")
total = sum(counter.values())
for chunk, count in counter.most_common(12):
pct = count / total * 100
f.write(f"{chunk:<6} {count:6} ({pct:.2f}%)\n")
f.write("\n")
# -----------------------------
# 8) Save Start/End pairs per line (TXT)
# -----------------------------
with open(OUTPUT_FILE_LINES, "w", encoding="utf-8") as f:
f.write("Line-Start → Line-End Chunk Pairs\n")
f.write("-" * 50 + "\n")
for (start, end), count in line_pairs.most_common():
f.write(f"{start:<8} → {end:<8} {count}x\n")
# -----------------------------
# 8b) Save Start/End pairs also as CSV
# -----------------------------
with open(OUTPUT_FILE_LINES_CSV, "w", encoding="utf-8") as f:
f.write("start_chunk,end_chunk,count\n")
for (start, end), count in line_pairs.most_common():
f.write(f"{start},{end},{count}\n")
# -----------------------------
# 9) Console output
# -----------------------------
print_top_chunks(word_start, "Top 12 Word-Start Chunks")
print_top_chunks(word_mid, "Top 12 Word-Mid Chunks")
print_top_chunks(word_end, "Top 12 Word-End Chunks")
print(f"\nDone! Chunked text saved in {OUTPUT_FILE_CHUNKS}")
print(f"Top chunks saved in {OUTPUT_FILE_TOP_CHUNKS}")
print(f"Line start/end pairs saved in {OUTPUT_FILE_LINES}")
print(f"CSV start/end pairs saved in {OUTPUT_FILE_LINES_CSV}")Here are the twelve most frequent chunks (start/middle/end), each listed
Frequent pairs (start chunk first word of the line, end chunk last word of the line)
RF1a-n-x7_line_start_end_pairs.xlsx (Size: 29.65 KB / Downloads: 8)
