This is an important statistic for assessing the quality of pseudo-Voynichese, so here is the Java code in case anyone else wants to try.
Code:
/*
* Edit distance calculation shamelessly borrowed from https://rosettacode.org/wiki/Levenshtein_distance#Java
* Example of use: java MDistBetweenLines text.txt 250 > text_dist.txt
*/
import java.io.*;
import java.util.*;
public class MDistBetweenLines {
static int maxLineDistance;
static ArrayList<ArrayList<String>> lineList = new ArrayList();
public static void main(String args[]) {
try {
if (args.length == 2) {
maxLineDistance = Integer.parseInt(args[1]);
loadFile(new File(args[0]));
processLines();
}
}
catch (Exception e) {
e.printStackTrace();
}
}
static void loadFile(File f) throws Exception {
InputStreamReader isreader = new InputStreamReader(new FileInputStream(f), "UTF-8");
BufferedReader reader = new BufferedReader(isreader);
String line;
while((line = reader.readLine()) != null) {
if (!line.isEmpty()) {
String[] words = line.toLowerCase().replaceAll("\\p{P}", " ").split("[0-9 \t]+");
ArrayList<String> wordList = new ArrayList<String>();
for (String word: words) {
if (!word.isEmpty())
wordList.add(word);
}
lineList.add(wordList);
}
}
reader.close();
System.err.println("Loaded " + lineList.size() + " lines");
}
static void processLines() {
for (int lineDistance = 1; lineDistance <= maxLineDistance; lineDistance++) {
int nSum = 0, totalDist = 0;
for (int iLine = 0; iLine < lineList.size() - lineDistance; iLine++) {
ArrayList<String> wordList1 = lineList.get(iLine);
ArrayList<String> wordList2 = lineList.get(iLine + lineDistance);
for (String word1 : wordList1) {
for (String word2 : wordList2) {
totalDist += distance(word1, word2);
nSum++;
}
}
}
System.out.println(lineDistance + "\t" + (double)totalDist/nSum);
}
}
public static int distance(String a, String b) {
int [] costs = new int [b.length() + 1];
for (int j = 0; j < costs.length; j++)
costs[j] = j;
for (int i = 1; i <= a.length(); i++) {
costs[0] = i;
int nw = i - 1;
for (int j = 1; j <= b.length(); j++) {
int cj = Math.min(1 + Math.min(costs[j], costs[j - 1]), a.charAt(i - 1) == b.charAt(j - 1) ? nw : nw + 1);
nw = costs[j];
costs[j] = cj;
}
}
return costs[b.length()];
}
}