Skip to content

Convert Hunspell Dictionary affix directive parsing else-if chain to … #16422

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rajat315315 wants to merge 1 commit into
base: main
Choose a base branch
from
+129 −141
Open

Convert Hunspell Dictionary affix directive parsing else-if chain to … #16422

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
270 changes: 129 additions & 141 deletions lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,134 +356,122 @@ private void readAffixFile(InputStream affixStream, CharsetDecoder decoder, Flag
if (line.isEmpty()) continue;

String firstWord = line.split("\\s")[0];
// TODO: convert to a switch?
if ("AF".equals(firstWord)) {
parseAlias(line);
} else if ("AM".equals(firstWord)) {
parseMorphAlias(line);
} else if ("PFX".equals(firstWord)) {
parseAffix(
prefixes, prefixContFlags, line, reader, PREFIX, seenPatterns, seenStrips, flags);
} else if ("SFX".equals(firstWord)) {
parseAffix(
suffixes, suffixContFlags, line, reader, SUFFIX, seenPatterns, seenStrips, flags);
} else if (line.equals("COMPLEXPREFIXES")) {
complexPrefixes =
true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix
} else if ("CIRCUMFIX".equals(firstWord)) {
circumfix = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("KEEPCASE".equals(firstWord)) {
keepcase = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("FORCEUCASE".equals(firstWord)) {
forceUCase = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("NEEDAFFIX".equals(firstWord) || "PSEUDOROOT".equals(firstWord)) {
needaffix = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("ONLYINCOMPOUND".equals(firstWord)) {
onlyincompound = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("CHECKSHARPS".equals(firstWord)) {
checkSharpS = true;
} else if ("IGNORE".equals(firstWord)) {
ignore = singleArgument(reader, line).toCharArray();
Arrays.sort(ignore);
} else if ("ICONV".equals(firstWord) || "OCONV".equals(firstWord)) {
int num = parseNum(reader, line);
ConvTable res = parseConversions(reader, num);
if (line.startsWith("I")) {
iconv = res;
} else {
oconv = res;
switch (firstWord) {
case "AF" -> parseAlias(line);
case "AM" -> parseMorphAlias(line);
case "PFX" ->
parseAffix(
prefixes, prefixContFlags, line, reader, PREFIX, seenPatterns, seenStrips, flags);
case "SFX" ->
parseAffix(
suffixes, suffixContFlags, line, reader, SUFFIX, seenPatterns, seenStrips, flags);
case "COMPLEXPREFIXES" ->
complexPrefixes =
true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix
case "CIRCUMFIX" -> circumfix = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "KEEPCASE" -> keepcase = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "FORCEUCASE" -> forceUCase = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "NEEDAFFIX", "PSEUDOROOT" ->
needaffix = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "ONLYINCOMPOUND" ->
onlyincompound = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "CHECKSHARPS" -> checkSharpS = true;
case "IGNORE" -> {
ignore = singleArgument(reader, line).toCharArray();
Arrays.sort(ignore);
}
} else if ("FULLSTRIP".equals(firstWord)) {
fullStrip = true;
} else if ("LANG".equals(firstWord)) {
language = singleArgument(reader, line);
this.alternateCasing = hasLanguage("tr", "az");
} else if ("BREAK".equals(firstWord)) {
breaks = parseBreaks(reader, line);
} else if ("WORDCHARS".equals(firstWord)) {
wordChars = firstArgument(reader, line);
} else if ("TRY".equals(firstWord)) {
tryChars = firstArgument(reader, line);
} else if ("REP".equals(firstWord)) {
if (tolerateRepRuleCountMismatches()) {
String[] parts = splitBySpace(reader, line, 2, Integer.MAX_VALUE);
// ignore REP N, as actual N may be incorrect
if (parts.length >= 3) {
repTable.add(new RepEntry(parts[1], parts[2]));
case "ICONV", "OCONV" -> {
int num = parseNum(reader, line);
ConvTable res = parseConversions(reader, num);
if (line.startsWith("I")) {
iconv = res;
} else {
oconv = res;
}
} else {
}
case "FULLSTRIP" -> fullStrip = true;
case "LANG" -> {
language = singleArgument(reader, line);
this.alternateCasing = hasLanguage("tr", "az");
}
case "BREAK" -> breaks = parseBreaks(reader, line);
case "WORDCHARS" -> wordChars = firstArgument(reader, line);
case "TRY" -> tryChars = firstArgument(reader, line);
case "REP" -> {
if (tolerateRepRuleCountMismatches()) {
String[] parts = splitBySpace(reader, line, 2, Integer.MAX_VALUE);
// ignore REP N, as actual N may be incorrect
if (parts.length >= 3) {
repTable.add(new RepEntry(parts[1], parts[2]));
}
} else {
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
String[] parts = splitBySpace(reader, reader.readLine(), 3, Integer.MAX_VALUE);
repTable.add(new RepEntry(parts[1], parts[2]));
}
}
}
case "MAP" -> {
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
String[] parts = splitBySpace(reader, reader.readLine(), 3, Integer.MAX_VALUE);
repTable.add(new RepEntry(parts[1], parts[2]));
mapTable.add(parseMapEntry(reader, reader.readLine()));
}
}
} else if ("MAP".equals(firstWord)) {
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
mapTable.add(parseMapEntry(reader, reader.readLine()));
case "KEY" -> neighborKeyGroups = singleArgument(reader, line).split("\\|");
case "NOSPLITSUGS" -> enableSplitSuggestions = false;
case "MAXNGRAMSUGS" -> maxNGramSuggestions = Integer.parseInt(singleArgument(reader, line));
case "MAXDIFF" -> {
int i = Integer.parseInt(singleArgument(reader, line));
if (i < 0 || i > 10) {
throw new ParseException("MAXDIFF should be between 0 and 10", reader.getLineNumber());
}
maxDiff = i;
}
case "ONLYMAXDIFF" -> onlyMaxDiff = true;
case "FORBIDDENWORD" ->
forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "NOSUGGEST" ->
noSuggest = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "SUBSTANDARD" ->
subStandard = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "COMPOUNDMIN" -> compoundMin = Math.max(1, parseNum(reader, line));
case "COMPOUNDWORDMAX" -> compoundMax = Math.max(1, parseNum(reader, line));
case "COMPOUNDRULE" -> compoundRules = parseCompoundRules(reader, parseNum(reader, line));
case "COMPOUNDFLAG" ->
compoundFlag = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "COMPOUNDBEGIN" ->
compoundBegin = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "COMPOUNDMIDDLE" ->
compoundMiddle = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "COMPOUNDEND" -> compoundEnd = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "COMPOUNDPERMITFLAG" ->
compoundPermit = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "COMPOUNDFORBIDFLAG" ->
compoundForbid = flagParsingStrategy.parseFlag(singleArgument(reader, line));
case "CHECKCOMPOUNDCASE" -> checkCompoundCase = true;
case "CHECKCOMPOUNDDUP" -> checkCompoundDup = true;
case "CHECKCOMPOUNDREP" -> checkCompoundRep = true;
case "CHECKCOMPOUNDTRIPLE" -> checkCompoundTriple = true;
case "SIMPLIFIEDTRIPLE" -> simplifiedTriple = true;
case "CHECKCOMPOUNDPATTERN" -> {
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
checkCompoundPatterns.add(
new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));
}
}
} else if ("KEY".equals(firstWord)) {
neighborKeyGroups = singleArgument(reader, line).split("\\|");
} else if ("NOSPLITSUGS".equals(firstWord)) {
enableSplitSuggestions = false;
} else if ("MAXNGRAMSUGS".equals(firstWord)) {
maxNGramSuggestions = Integer.parseInt(singleArgument(reader, line));
} else if ("MAXDIFF".equals(firstWord)) {
int i = Integer.parseInt(singleArgument(reader, line));
if (i < 0 || i > 10) {
throw new ParseException("MAXDIFF should be between 0 and 10", reader.getLineNumber());
case "SET" ->
checkCriticalDirectiveSame(
"SET", reader, decoder.charset(), getDecoder(singleArgument(reader, line)).charset());
case "FLAG" -> {
FlagParsingStrategy strategy = getFlagParsingStrategy(line, decoder.charset());
checkCriticalDirectiveSame(
"FLAG", reader, flagParsingStrategy.getClass(), strategy.getClass());
}
maxDiff = i;
} else if ("ONLYMAXDIFF".equals(firstWord)) {
onlyMaxDiff = true;
} else if ("FORBIDDENWORD".equals(firstWord)) {
forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("NOSUGGEST".equals(firstWord)) {
noSuggest = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("SUBSTANDARD".equals(firstWord)) {
subStandard = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDMIN".equals(firstWord)) {
compoundMin = Math.max(1, parseNum(reader, line));
} else if ("COMPOUNDWORDMAX".equals(firstWord)) {
compoundMax = Math.max(1, parseNum(reader, line));
} else if ("COMPOUNDRULE".equals(firstWord)) {
compoundRules = parseCompoundRules(reader, parseNum(reader, line));
} else if ("COMPOUNDFLAG".equals(firstWord)) {
compoundFlag = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDBEGIN".equals(firstWord)) {
compoundBegin = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDMIDDLE".equals(firstWord)) {
compoundMiddle = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDEND".equals(firstWord)) {
compoundEnd = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDPERMITFLAG".equals(firstWord)) {
compoundPermit = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDFORBIDFLAG".equals(firstWord)) {
compoundForbid = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("CHECKCOMPOUNDCASE".equals(firstWord)) {
checkCompoundCase = true;
} else if ("CHECKCOMPOUNDDUP".equals(firstWord)) {
checkCompoundDup = true;
} else if ("CHECKCOMPOUNDREP".equals(firstWord)) {
checkCompoundRep = true;
} else if ("CHECKCOMPOUNDTRIPLE".equals(firstWord)) {
checkCompoundTriple = true;
} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {
simplifiedTriple = true;
} else if ("CHECKCOMPOUNDPATTERN".equals(firstWord)) {
int count = parseNum(reader, line);
for (int i = 0; i < count; i++) {
checkCompoundPatterns.add(
new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));
default -> {
// ignore unknown directives
}
} else if ("SET".equals(firstWord)) {
checkCriticalDirectiveSame(
"SET", reader, decoder.charset(), getDecoder(singleArgument(reader, line)).charset());
} else if ("FLAG".equals(firstWord)) {
FlagParsingStrategy strategy = getFlagParsingStrategy(line, decoder.charset());
checkCriticalDirectiveSame(
"FLAG", reader, flagParsingStrategy.getClass(), strategy.getClass());
}
}

Expand Down Expand Up @@ -855,16 +843,20 @@ private void readConfig(InputStream stream, Charset streamCharset)
if (line.isBlank()) continue;

String firstWord = line.split("\\s")[0];
if ("SET".equals(firstWord)) {
decoder = getDecoder(singleArgument(reader, line));
charsetFound = true;
} else if ("FLAG".equals(firstWord)) {
// Preserve the flag line for parsing later since we need the decoder's charset
// and just in case they come out of order.
flagLine = line;
flagFound = true;
} else {
continue;
switch (firstWord) {
case "SET" -> {
decoder = getDecoder(singleArgument(reader, line));
charsetFound = true;
}
case "FLAG" -> {
// Preserve the flag line for parsing later since we need the decoder's charset
// and just in case they come out of order.
flagLine = line;
flagFound = true;
}
default -> {
continue;
}
}

if (charsetFound && flagFound) {
Expand Down Expand Up @@ -936,18 +928,14 @@ static FlagParsingStrategy getFlagParsingStrategy(String flagLine, Charset chars
}
String flagType = parts[1];

if ("num".equals(flagType)) {
return new NumFlagParsingStrategy();
} else if ("UTF-8".equals(flagType)) {
if (DEFAULT_CHARSET.equals(charset)) {
return new DefaultAsUtf8FlagParsingStrategy();
}
return new SimpleFlagParsingStrategy();
} else if ("long".equals(flagType)) {
return new DoubleASCIIFlagParsingStrategy();
}

throw new IllegalArgumentException("Unknown flag type: " + flagType);
return switch (flagType) {
case "num" -> new NumFlagParsingStrategy();
case "UTF-8" -> DEFAULT_CHARSET.equals(charset)
? new DefaultAsUtf8FlagParsingStrategy()
: new SimpleFlagParsingStrategy();
case "long" -> new DoubleASCIIFlagParsingStrategy();
default -> throw new IllegalArgumentException("Unknown flag type: " + flagType);
};
}

private static final char FLAG_SEPARATOR = 0x1f; // flag separator after escaping
Expand Down