Commit 9a67d388 authored by Linda Freienthal's avatar Linda Freienthal
Browse files

Initial commit

parent d29510ae
Pipeline #7 canceled with stages
File added
File added
File added
trikk
triki
triki
triki
triki
triki
trike
triki
lapsiku
õnnelik
õnneliku
õnneliku
õnneliku
õnneliku
õnnelik
õnnelik
õnneliku
õnnelik
õnnelik
hobune
hobuse
hobu
hobuse
hobuse
hobus
hobuse
hobus
kana
kanali
kanalit
kanali
kanali
kanali
kanale
kanali
kanalei
rata
rata
rata
rata
rata
ratas
rata
ratas
kott
koti
koti
koti
koti
koti
kot
koti
This diff is collapsed.
/*
* Authors:
* - Assem Chelli, < assem [dot] ch [at] gmail >
* - Abdelkrim Aries <ab [underscore] aries [at] esi [dot] dz>
*
*/
stringescapes { }
/* the Arabic letters in Unicode */
// Hamza
stringdef o '{U+0621}' // Hamza
stringdef ao '{U+0623}' // Hamza above Alef
stringdef ao_ '{U+0625}' // Hamza below Alef
stringdef a~ '{U+0622}' // Alef madda
stringdef wo '{U+0624}' // Hamza above waw
stringdef yo '{U+0626}' // Hamza above yeh
// Letters
stringdef a '{U+0627}' // Alef
stringdef a_ '{U+0649}' // Alef Maksura
stringdef b '{U+0628}' // Beh
stringdef t_ '{U+0629}' // Teh_Marbuta
stringdef t '{U+062A}' // Teh
stringdef th '{U+062B}' // Theh
stringdef j '{U+062C}' // Jeem
stringdef h '{U+062D}' // Hah
stringdef x '{U+062E}' // Khah
stringdef d '{U+062F}' // Dal
stringdef dz '{U+0630}' // Thal
stringdef r '{U+0631}' // Reh
stringdef z '{U+0632}' // Zain
stringdef s '{U+0633}' // Seen
stringdef sh '{U+0634}' // Sheen
stringdef c '{U+0635}' // Sad
stringdef dh '{U+0636}' // Dad
stringdef tt '{U+0637}' // Tah
stringdef zh '{U+0638}' // Zah
stringdef i '{U+0639}' // Ain
stringdef gh '{U+063A}' // Ghain
stringdef f '{U+0641}' // Feh
stringdef q '{U+0642}' // Qaf
stringdef k '{U+0643}' // Kaf
stringdef l '{U+0644}' // Lam
stringdef m '{U+0645}' // Meem
stringdef n '{U+0646}' // Noon
stringdef e '{U+0647}' // Heh
stringdef w '{U+0648}' // Waw
stringdef y '{U+064A}' // Yeh
// Diacritics
stringdef aan '{U+064B}' // FatHatan
stringdef uun '{U+064C}' // Dammatan
stringdef iin '{U+064D}' // Kasratan
stringdef aa '{U+064E}' // FatHa
stringdef uu '{U+064F}' // Damma
stringdef ii '{U+0650}' // Kasra
stringdef oo '{U+0652}' // Sukun
stringdef ~ '{U+0651}' // Shadda
// Hindu–Arabic numerals
stringdef 0 '{U+0660}'
stringdef 1 '{U+0661}'
stringdef 2 '{U+0662}'
stringdef 3 '{U+0663}'
stringdef 4 '{U+0664}'
stringdef 5 '{U+0665}'
stringdef 6 '{U+0666}'
stringdef 7 '{U+0667}'
stringdef 8 '{U+0668}'
stringdef 9 '{U+0669}'
// Kasheeda
stringdef _ '{U+0640}' // Kasheeda, Tatweel
// Shaped forms
stringdef o1 '{U+FE80}' // HAMZA
stringdef ao1 '{U+FE83}' // ALEF_HAMZA_ABOVE
stringdef ao2 '{U+FE84}' // ALEF_HAMZA_ABOVE
stringdef ao_1 '{U+FE87}' // ALEF_HAMZA_BELOW
stringdef ao_2 '{U+FE88}' // ALEF_HAMZA_BELOW
stringdef yo1 '{U+FE8B}' // YEH_HAMZA
stringdef yo2 '{U+FE8C}' // YEH_HAMZA
stringdef yo3 '{U+FE89}' // YEH_HAMZA
stringdef yo4 '{U+FE8A}' // YEH_HAMZA
stringdef a~1 '{U+FE81}' // ALEF_MADDA
stringdef a~2 '{U+FE82}' // ALEF_MADDA
stringdef wo1 '{U+FE85}' // WAW_HAMZA
stringdef wo2 '{U+FE86}' // WAW_HAMZA
stringdef a1 '{U+FE8D}' // ALEF
stringdef a2 '{U+FE8E}' // ALEF
stringdef b1 '{U+FE8F}' // BEH
stringdef b2 '{U+FE90}' // BEH
stringdef b3 '{U+FE91}' // BEH
stringdef b4 '{U+FE92}' // BEH
stringdef t_1 '{U+FE93}' // TEH_MARBUTA
stringdef t_2 '{U+FE94}' // TEH_MARBUTA
stringdef t1 '{U+FE97}' // TEH
stringdef t2 '{U+FE98}' // TEH
stringdef t3 '{U+FE95}' // TEH
stringdef t4 '{U+FE96}' // TEH
stringdef th1 '{U+FE9B}' // THEH
stringdef th2 '{U+FE9C}' // THEH
stringdef th3 '{U+FE9A}' // THEH
stringdef th4 '{U+FE99}' // THEH
stringdef j1 '{U+FE9F}' // JEEM
stringdef j2 '{U+FEA0}' // JEEM
stringdef j3 '{U+FE9D}' // JEEM
stringdef j4 '{U+FE9E}' // JEEM
stringdef h1 '{U+FEA3}' // HAH
stringdef h2 '{U+FEA4}' // HAH
stringdef h3 '{U+FEA1}' // HAH
stringdef h4 '{U+FEA2}' // HAH
stringdef x1 '{U+FEA7}' // KHAH
stringdef x2 '{U+FEA8}' // KHAH
stringdef x3 '{U+FEA5}' // KHAH
stringdef x4 '{U+FEA6}' // KHAH
stringdef d1 '{U+FEA9}' // DAL
stringdef d2 '{U+FEAA}' // DAL
stringdef dz1 '{U+FEAB}' // THAL
stringdef dz2 '{U+FEAC}' // THAL
stringdef r1 '{U+FEAD}' // REH
stringdef r2 '{U+FEAE}' // REH
stringdef z1 '{U+FEAF}' // ZAIN
stringdef z2 '{U+FEB0}' // ZAIN
stringdef s1 '{U+FEB3}' // SEEN
stringdef s2 '{U+FEB4}' // SEEN
stringdef s3 '{U+FEB1}' // SEEN
stringdef s4 '{U+FEB2}' // SEEN
stringdef sh1 '{U+FEB7}' // SHEEN
stringdef sh2 '{U+FEB8}' // SHEEN
stringdef sh3 '{U+FEB5}' // SHEEN
stringdef sh4 '{U+FEB6}' // SHEEN
stringdef c1 '{U+FEBB}' // SAD
stringdef c2 '{U+FEBC}' // SAD
stringdef c3 '{U+FEB9}' // SAD
stringdef c4 '{U+FEBA}' // SAD
stringdef dh1 '{U+FEBF}' // DAD
stringdef dh2 '{U+FEC0}' // DAD
stringdef dh3 '{U+FEBD}' // DAD
stringdef dh4 '{U+FEBE}' // DAD
stringdef tt1 '{U+FEC3}' // TAH
stringdef tt2 '{U+FEC4}' // TAH
stringdef tt3 '{U+FEC1}' // TAH
stringdef tt4 '{U+FEC2}' // TAH
stringdef zh1 '{U+FEC7}' // ZAH
stringdef zh2 '{U+FEC8}' // ZAH
stringdef zh3 '{U+FEC5}' // ZAH
stringdef zh4 '{U+FEC6}' // ZAH
stringdef i1 '{U+FECB}' // AIN
stringdef i2 '{U+FECC}' // AIN
stringdef i3 '{U+FEC9}' // AIN
stringdef i4 '{U+FECA}' // AIN
stringdef gh1 '{U+FECF}' // GHAIN
stringdef gh2 '{U+FED0}' // GHAIN
stringdef gh3 '{U+FECD}' // GHAIN
stringdef gh4 '{U+FECE}' // GHAIN
stringdef f1 '{U+FED3}' // FEH
stringdef f2 '{U+FED4}' // FEH
stringdef f3 '{U+FED1}' // FEH
stringdef f4 '{U+FED2}' // FEH
stringdef q1 '{U+FED7}' // QAF
stringdef q2 '{U+FED8}' // QAF
stringdef q3 '{U+FED5}' // QAF
stringdef q4 '{U+FED6}' // QAF
stringdef k1 '{U+FEDB}' // KAF
stringdef k2 '{U+FEDC}' // KAF
stringdef k3 '{U+FED9}' // KAF
stringdef k4 '{U+FEDA}' // KAF
stringdef l1 '{U+FEDF}' // LAM
stringdef l2 '{U+FEE0}' // LAM
stringdef l3 '{U+FEDD}' // LAM
stringdef l4 '{U+FEDE}' // LAM
stringdef m1 '{U+FEE3}' // MEEM
stringdef m2 '{U+FEE4}' // MEEM
stringdef m3 '{U+FEE1}' // MEEM
stringdef m4 '{U+FEE2}' // MEEM
stringdef n1 '{U+FEE7}' // NOON
stringdef n2 '{U+FEE8}' // NOON
stringdef n3 '{U+FEE5}' // NOON
stringdef n4 '{U+FEE6}' // NOON
stringdef e1 '{U+FEEB}' // HEH
stringdef e2 '{U+FEEC}' // HEH
stringdef e3 '{U+FEE9}' // HEH
stringdef e4 '{U+FEEA}' // HEH
stringdef w1 '{U+FEED}' // WAW
stringdef w2 '{U+FEEE}' // WAW
stringdef a_1 '{U+FEEF}' // ALEF_MAKSURA
stringdef a_2 '{U+FEF0}' // ALEF_MAKSURA
stringdef y1 '{U+FEF3}' // YEH
stringdef y2 '{U+FEF4}' // YEH
stringdef y3 '{U+FEF1}' // YEH
stringdef y4 '{U+FEF2}' // YEH
// Ligatures Lam-Alef
stringdef la '{U+FEFB}' // LAM_ALEF
stringdef la2 '{U+FEFC}' // LAM_ALEF
stringdef lao '{U+FEF7}' // LAM_ALEF_HAMZA_ABOVE
stringdef lao2 '{U+FEF8}' // LAM_ALEF_HAMZA_ABOVE
stringdef lao_ '{U+FEF9}' // LAM_ALEF_HAMZA_BELOW
stringdef lao_2 '{U+FEFA}' // LAM_ALEF_HAMZA_BELOW
stringdef la~ '{U+FEF5}' // LAM_ALEF_MADDA_ABOVE
stringdef la~2 '{U+FEF6}' // LAM_ALEF_MADDA_ABOVE
booleans (
is_noun
is_verb
is_defined
)
routines (
Prefix_Step1
Prefix_Step2
Prefix_Step3a_Noun
Prefix_Step3b_Noun
Prefix_Step3_Verb
Prefix_Step4_Verb
Suffix_All_alef_maqsura
Suffix_Noun_Step1a
Suffix_Noun_Step1b
Suffix_Noun_Step2a
Suffix_Noun_Step2b
Suffix_Noun_Step2c1
Suffix_Noun_Step2c2
Suffix_Noun_Step3
Suffix_Verb_Step1
Suffix_Verb_Step2a
Suffix_Verb_Step2b
Suffix_Verb_Step2c
Normalize_post
Normalize_pre
Checks1
)
externals ( stem )
groupings ( )
// Normalizations
define Normalize_pre as (
do repeat (
(
[substring] among (
'{aan}' '{uun}' '{iin}' '{aa}' '{uu}' '{ii}' '{oo}' '{~}'( delete ) // strip vocalization
'{_}' ( delete ) // strip kasheeda
// Hindu–Arabic numerals
'{0}' ( <- '0')
'{1}' ( <- '1')
'{2}' ( <- '2')
'{3}' ( <- '3')
'{4}' ( <- '4')
'{5}' ( <- '5')
'{6}' ( <- '6')
'{7}' ( <- '7')
'{8}' ( <- '8')
'{9}' ( <- '9')
// Shaped forms
'{o1}' ( <- '{o}' ) // HAMZA
'{ao1}' '{ao2}' ( <- '{ao}' ) // ALEF_HAMZA_ABOVE
'{ao_1}' '{ao_2}' ( <- '{ao_}' ) // ALEF_HAMZA_BELOW
'{yo1}' '{yo2}' '{yo3}' '{yo4}' ( <- '{yo}' ) // YEH_HAMZA
'{a~1}' '{a~2}'( <- '{a~}' ) // ALEF_MADDA
'{wo1}' '{wo2}'( <- '{wo}' ) // WAW_HAMZA
'{a1}' '{a2}' ( <- '{a}' ) // ALEF
'{b1}' '{b2}' '{b3}' '{b4}' ( <- '{b}' ) // BEH
'{t_1}' '{t_2}' ( <- '{t_}' ) // TEH_MARBUTA
'{t1}' '{t2}' '{t3}' '{t4}' ( <- '{t}' ) // TEH
'{th1}' '{th2}' '{th3}' '{th4}' ( <- '{th}' ) // THEH
'{j1}' '{j2}' '{j3}' '{j4}'( <- '{j}' ) // JEEM
'{h1}' '{h2}' '{h3}' '{h4}' ( <- '{h}' ) // HAH
'{x1}' '{x2}' '{x3}' '{x4}'( <- '{x}' ) // KHAH
'{d1}' '{d2}' ( <- '{d}' ) // DAL
'{dz1}''{dz2}' ( <- '{dz}' ) // THAL
'{r1}' '{r2}'( <- '{r}' ) // REH
'{z1}' '{z2}' ( <- '{z}' ) // ZAIN
'{s1}' '{s2}' '{s3}' '{s4}'( <- '{s}' ) // SEEN
'{sh1}' '{sh2}' '{sh3}' '{sh4}' ( <- '{sh}' ) // SHEEN
'{c1}' '{c2}' '{c3}' '{c4}'( <- '{c}' ) // SAD
'{dh1}' '{dh2}' '{dh3}' '{dh4}'( <- '{dh}' ) // DAD
'{tt1}' '{tt2}' '{tt3}' '{tt4}' ( <- '{tt}' ) // TAH
'{zh1}' '{zh2}' '{zh3}' '{zh4}'( <- '{zh}' ) // ZAH
'{i1}' '{i2}' '{i3}' '{i4}'( <- '{i}' ) // AIN
'{gh1}' '{gh2}' '{gh3}' '{gh4}'( <- '{gh}' ) // GHAIN
'{f1}' '{f2}' '{f3}' '{f4}' ( <- '{f}' ) // FEH
'{q1}' '{q2}' '{q3}' '{q4}' ( <- '{q}' ) // QAF
'{k1}' '{k2}' '{k3}' '{k4}'( <- '{k}' ) // KAF
'{l1}' '{l2}' '{l3}' '{l4}'( <- '{l}' ) // LAM
'{m1}' '{m2}' '{m3}' '{m4}' ( <- '{m}' ) // MEEM
'{n1}' '{n2}' '{n3}' '{n4}'( <- '{n}' ) // NOON
'{e1}' '{e2}' '{e3}' '{e4}' ( <- '{e}' ) // HEH
'{w1}' '{w2}' ( <- '{w}' ) // WAW
'{a_1}' '{a_2}' ( <- '{a_}' ) // ALEF_MAKSURA
'{y1}' '{y2}' '{y3}' '{y4}' ( <- '{y}' ) // YEH
// Ligatures Lam-Alef
'{la}' '{la2}' (<- '{l}{a}')
'{lao}' '{lao2}' (<- '{l}{ao}')
'{lao_}' '{lao_2}' (<- '{l}{ao_}')
'{la~}' '{la~2}' (<- '{l}{a~}')
)
)
or
next
)
)
define Normalize_post as (
do (
// normalize last hamza
backwards (
[substring] among (
'{ao}''{ao_}' '{a~}' ( <- '{o}')
'{wo}' ( <- '{o}')
'{yo}' ( <- '{o}')
)
)
)
do repeat (
(
// normalize other hamza's
[substring] among (
'{ao}''{ao_}' '{a~}' ( <- '{a}')
'{wo}' ( <- '{w}')
'{yo}' ( <- '{y}')
)
)
or
next
)
)
// Checks
define Checks1 as (
[substring] among (
'{b}{a}{l}' '{k}{a}{l}' ($(len > 4) set is_noun unset is_verb set is_defined)
'{l}{l}' '{a}{l}' ($(len > 3) set is_noun unset is_verb set is_defined)
)
)
//prefixes
define Prefix_Step1 as (
[substring] among (
'{ao}{ao}' ($(len > 3) <- '{ao}' )
'{ao}{a~}' ($(len > 3) <- '{a~}' )
'{ao}{wo}' ($(len > 3) <- '{ao}' )
'{ao}{a}' ($(len > 3) <- '{a}' )
'{ao}{ao_}' ($(len > 3) <- '{ao_}' )
// '{ao}' ($(len > 3) delete) //rare case
)
)
define Prefix_Step2 as (
not '{f}{a}'
not '{w}{a}'
[substring] among (
'{f}' ($(len > 3) delete)
'{w}' ($(len > 3) delete)
)
)
define Prefix_Step3a_Noun as ( // it is noun and defined
[substring] among (
'{b}{a}{l}' '{k}{a}{l}' ($(len > 5) delete)
'{l}{l}' '{a}{l}' ($(len > 4) delete)
)
)
define Prefix_Step3b_Noun as ( // probably noun and defined
not '{b}{a}' // exception
[substring] among (
'{b}' ($(len > 3) delete)
// '{k}' '{l}' ($(len > 3) delete) // BUG: cause confusion
'{b}{b}' ($(len > 3) <- '{b}' )
'{k}{k}' ($(len > 3) <- '{k}' )
)
)
define Prefix_Step3_Verb as (
[substring] among (
//'{s}' ($(len > 4) delete)// BUG: cause confusion
'{s}{y}' ($(len > 4) <- '{y}' )
'{s}{t}' ($(len > 4) <- '{t}')
'{s}{n}' ($(len > 4) <- '{n}')
'{s}{ao}' ($(len > 4) <- '{ao}')
)
)
define Prefix_Step4_Verb as (
[substring] among (
'{y}{s}{t}' '{n}{s}{t}' '{t}{s}{t}' ($(len > 4) set is_verb unset is_noun <- '{a}{s}{t}' )
)
)
// suffixes
backwardmode (
define Suffix_Noun_Step1a as (
[substring] among (
'{y}' '{k}' '{e}' ($(len >= 4) delete)
'{n}{a}' '{k}{m}' '{e}{a}' '{e}{n}' '{e}{m}' ($(len >= 5) delete)
'{k}{m}{a}' '{e}{m}{a}' ($(len >= 6) delete)
)
)
define Suffix_Noun_Step1b as (
[substring] among (
'{n}' ($(len > 5) delete)
)
)
define Suffix_Noun_Step2a as (
[substring] among (
'{a}' '{y}' '{w}' ($(len > 4) delete)
)
)
define Suffix_Noun_Step2b as (
[substring] among (
'{a}{t}' ($(len >= 5) delete)
)
)
define Suffix_Noun_Step2c1 as (
[substring] among (
'{t}' ($(len >= 4) delete)
)
)
define Suffix_Noun_Step2c2 as ( // feminine t_
[substring] among (
'{t_}' ($(len >= 4) delete)
)
)
define Suffix_Noun_Step3 as ( // ya' nisbiya
[substring] among (
'{y}' ($(len >= 3) delete)
)
)
define Suffix_Verb_Step1 as (
[substring] among (
'{e}' '{k}' ($(len >= 4) delete)
'{n}{y}' '{n}{a}' '{e}{a}' '{e}{m}' '{e}{n}' '{k}{m}' '{k}{n}' ($(len >= 5) delete)
'{e}{m}{a}' '{k}{m}{a}' '{k}{m}{w}'($(len >= 6) delete)
)
)
define Suffix_Verb_Step2a as (
[substring] among (
'{t}' ($(len >= 4) delete)
'{a}' '{n}' '{y}' ($(len >= 4) delete)
'{n}{a}' '{t}{a}' '{t}{n}' ($(len >= 5) delete)// past
'{a}{n}' '{w}{n}' '{y}{n}' ($(len > 5) delete) // present
'{t}{m}{a}' ($(len >= 6) delete)
)
)
define Suffix_Verb_Step2b as (
[substring] among (
'{w}{a}' '{t}{m}' ($(len >= 5) delete)
)
)
define Suffix_Verb_Step2c as (
[substring] among (
'{w}' ($(len >= 4) delete)
'{t}{m}{w}' ($(len >= 6) delete)
)
)
define Suffix_All_alef_maqsura as (
[substring] among (
'{a_}' ( <- '{y}' ) // spell error
// '{a_}' ( delete ) // if noun > 3
// '{a_}' ( <- '{a}') // if verb
)
)
)
define stem as (
// set initial values
set is_noun
set is_verb
unset is_defined
// guess type and properties
do Checks1
// normalization pre-stemming
do Normalize_pre
backwards (
do (
//Suffixes for verbs
(
is_verb
(
(
(atleast 1 Suffix_Verb_Step1)
( Suffix_Verb_Step2a or Suffix_Verb_Step2c or next)
)
or Suffix_Verb_Step2b
or Suffix_Verb_Step2a
)
)
//Suffixes for nouns
or (
is_noun
(
try (
Suffix_Noun_Step2c2
or (not is_defined Suffix_Noun_Step1a (
Suffix_Noun_Step2a
or Suffix_Noun_Step2b
or Suffix_Noun_Step2c1
or next))
or (Suffix_Noun_Step1b (
Suffix_Noun_Step2a
or Suffix_Noun_Step2b
or Suffix_Noun_Step2c1))
or (not is_defined Suffix_Noun_Step2a)
or (Suffix_Noun_Step2b)
)
Suffix_Noun_Step3
)
)
// Suffixes for alef maqsura
or Suffix_All_alef_maqsura
)
)
//Prefixes
do (
try Prefix_Step1
try Prefix_Step2
( Prefix_Step3a_Noun
or (is_noun Prefix_Step3b_Noun)
or (is_verb try Prefix_Step3_Verb Prefix_Step4_Verb)
)
)
// normalization post-stemming
do Normalize_post
)
routines (
mark_regions
main_suffix
consonant_pair
other_suffix
undouble
)
externals ( stem )
strings ( ch )
integers ( p1 x )
groupings ( c v s_ending )
stringescapes {}
/* special characters */
stringdef ae '{U+00E6}'
stringdef ao '{U+00E5}'
stringdef o/ '{U+00F8}'
define c 'bcdfghjklmnpqrstvwxz'
define v 'aeiouy{ae}{ao}{o/}'