Duplicate from ai4bharat/IndicTrans-Indic2English

Co-authored-by: Julien Chaumond <[email protected]>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Gowtham.R
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,13 @@
+---
+title: IndicTrans-Indic2English
+emoji: 🌖
+colorFrom: green
+colorTo: green
+sdk: gradio
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: ai4bharat/IndicTrans-Indic2English
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

api/api.py

@@ -0,0 +1,152 @@
+import time
+
+import re
+from math import floor, ceil
+from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
+# from nltk.tokenize import sent_tokenize
+from flask import Flask, request, jsonify
+from flask_cors import CORS, cross_origin
+import webvtt
+from io import StringIO
+from mosestokenizer import MosesSentenceSplitter
+
+from indicTrans.inference.engine import Model
+from punctuate import RestorePuncts
+from indicnlp.tokenize.sentence_tokenize import sentence_split
+
+app = Flask(__name__)
+cors = CORS(app)
+app.config['CORS_HEADERS'] = 'Content-Type'
+
+indic2en_model = Model(expdir='models/v3/indic-en')
+en2indic_model = Model(expdir='models/v3/en-indic')
+m2m_model = Model(expdir='models/m2m')
+
+rpunct = RestorePuncts()
+
+indic_language_dict = {
+    'Assamese': 'as',
+    'Hindi' : 'hi',
+    'Marathi' : 'mr',
+    'Tamil' : 'ta',
+    'Bengali' : 'bn',
+    'Kannada' : 'kn',
+    'Oriya' : 'or',
+    'Telugu' : 'te',
+    'Gujarati' : 'gu',
+    'Malayalam' : 'ml',
+    'Punjabi' : 'pa',
+}
+
+splitter = MosesSentenceSplitter('en')
+
+def get_inference_params():
+    source_language = request.form['source_language']
+    target_language = request.form['target_language']
+
+    if source_language in indic_language_dict and target_language == 'English':
+        model = indic2en_model
+        source_lang = indic_language_dict[source_language]
+        target_lang = 'en'
+    elif source_language == 'English' and target_language in indic_language_dict:
+        model = en2indic_model
+        source_lang = 'en'
+        target_lang = indic_language_dict[target_language]
+    elif source_language in indic_language_dict and target_language in indic_language_dict:
+        model = m2m_model
+        source_lang = indic_language_dict[source_language]
+        target_lang = indic_language_dict[target_language]
+    
+    return model, source_lang, target_lang
+
+@app.route('/', methods=['GET'])
+def main():
+    return "IndicTrans API"
+
+@app.route('/supported_languages', methods=['GET'])
+@cross_origin()
+def supported_languages():
+    return jsonify(indic_language_dict)
+
+@app.route("/translate", methods=['POST'])
+@cross_origin()
+def infer_indic_en():
+    model, source_lang, target_lang = get_inference_params()
+    source_text = request.form['text']
+
+    start_time = time.time()
+    target_text = model.translate_paragraph(source_text, source_lang, target_lang)
+    end_time = time.time()
+    return {'text':target_text, 'duration':round(end_time-start_time, 2)}
+
+@app.route("/translate_vtt", methods=['POST'])
+@cross_origin()
+def infer_vtt_indic_en():
+    start_time = time.time()
+    model, source_lang, target_lang = get_inference_params()
+    source_text = request.form['text']
+    # vad_segments = request.form['vad_nochunk'] # Assuming it is an array of start & end timestamps
+
+    vad = webvtt.read_buffer(StringIO(source_text))
+    source_sentences = [v.text.replace('\r', '').replace('\n', ' ') for v in vad]
+
+    ## SUMANTH LOGIC HERE ##
+
+    # for each vad timestamp, do:
+    large_sentence = ' '.join(source_sentences) # only sentences in that time range
+    large_sentence = large_sentence.lower()
+    # split_sents = sentence_split(large_sentence, 'en')
+    # print(split_sents)
+
+    large_sentence = re.sub(r'[^\w\s]', '', large_sentence)
+    punctuated = rpunct.punctuate(large_sentence, batch_size=32)
+    end_time = time.time()
+    print("Time Taken for punctuation: {} s".format(end_time - start_time))
+    start_time = time.time()
+    split_sents = splitter([punctuated]) ### Please uncomment
+
+
+    # print(split_sents)
+    # output_sentence_punctuated = model.translate_paragraph(punctuated, source_lang, target_lang)
+    output_sents = model.batch_translate(split_sents, source_lang, target_lang)
+    # print(output_sents)
+    # output_sents = split_sents
+    # print(output_sents)
+    # align this to those range of source_sentences in `captions`
+
+    map_ = {split_sents[i] : output_sents[i] for i in range(len(split_sents))}
+    # print(map_)
+    punct_para = ' '.join(list(map_.keys()))
+    nmt_para = ' '.join(list(map_.values()))
+    nmt_words = nmt_para.split(' ')
+
+    len_punct = len(punct_para.split(' '))
+    len_nmt = len(nmt_para.split(' '))
+
+    start = 0
+    for i in range(len(vad)):
+        if vad[i].text == '':
+            continue
+
+        len_caption = len(vad[i].text.split(' '))
+        frac = (len_caption / len_punct)
+        # frac = round(frac, 2)
+
+        req_nmt_size = floor(frac * len_nmt)
+        # print(frac, req_nmt_size)
+
+        vad[i].text = ' '.join(nmt_words[start:start+req_nmt_size])
+        # print(vad[i].text)
+        # print(start, req_nmt_size)
+        start += req_nmt_size
+
+    end_time = time.time()
+    
+    print("Time Taken for translation: {} s".format(end_time - start_time))
+
+    # vad.save('aligned.vtt')
+
+    return {
+        'text': vad.content,
+        # 'duration':round(end_time-start_time, 2)
+    }

api/punctuate.py

@@ -0,0 +1,220 @@
+# -*- coding: utf-8 -*-
+# 💾⚙️🔮
+
+# taken from https://github.com/Felflare/rpunct/blob/master/rpunct/punctuate.py
+# modified to support batching during gpu inference
+
+
+__author__ = "Daulet N."
+__email__ = "[email protected]"
+
+import time
+import logging
+import webvtt
+import torch
+from io import StringIO
+from nltk.tokenize import sent_tokenize
+#from langdetect import detect
+from simpletransformers.ner import NERModel
+
+
+class RestorePuncts:
+    def __init__(self, wrds_per_pred=250):
+        self.wrds_per_pred = wrds_per_pred
+        self.overlap_wrds = 30
+        self.valid_labels = ['OU', 'OO', '.O', '!O', ',O', '.U', '!U', ',U', ':O', ';O', ':U', "'O", '-O', '?O', '?U']
+        self.model = NERModel("bert", "felflare/bert-restore-punctuation", labels=self.valid_labels,
+                              args={"silent": True, "max_seq_length": 512})
+        # use_cuda isnt working and this hack seems to load the model correctly to the gpu
+        self.model.device = torch.device("cuda:1")
+        # dummy punctuate to load the model onto gpu
+        self.punctuate("hello how are you")
+
+    def punctuate(self, text: str, batch_size:int=32, lang:str=''):
+        """
+        Performs punctuation restoration on arbitrarily large text.
+        Detects if input is not English, if non-English was detected terminates predictions.
+        Overrride by supplying `lang='en'`
+        
+        Args:
+            - text (str): Text to punctuate, can be few words to as large as you want.
+            - lang (str): Explicit language of input text.
+        """
+        #if not lang and len(text) > 10:
+        #    lang = detect(text)
+        #if lang != 'en':
+        #    raise Exception(F"""Non English text detected. Restore Punctuation works only for English.
+        #    If you are certain the input is English, pass argument lang='en' to this function.
+        #    Punctuate received: {text}""")
+
+        def chunks(L, n):
+            return [L[x : x + n] for x in range(0, len(L), n)]
+
+
+
+        # plit up large text into bert digestable chunks
+        splits = self.split_on_toks(text, self.wrds_per_pred, self.overlap_wrds)
+
+        texts = [i["text"] for i in splits]
+        batches = chunks(texts, batch_size)
+        preds_lst = []
+
+
+        for batch in batches:
+            batch_preds, _ = self.model.predict(batch)
+            preds_lst.extend(batch_preds)
+
+        
+        # predict slices
+        # full_preds_lst contains tuple of labels and logits
+        #full_preds_lst = [self.predict(i['text']) for i in splits]
+        # extract predictions, and discard logits
+        #preds_lst = [i[0][0] for i in full_preds_lst]
+        # join text slices
+        combined_preds = self.combine_results(text, preds_lst)
+        # create punctuated prediction
+        punct_text = self.punctuate_texts(combined_preds)
+        return punct_text
+
+    def predict(self, input_slice):
+        """
+        Passes the unpunctuated text to the model for punctuation.
+        """
+        predictions, raw_outputs = self.model.predict([input_slice])
+        return predictions, raw_outputs
+
+    @staticmethod
+    def split_on_toks(text, length, overlap):
+        """
+        Splits text into predefined slices of overlapping text with indexes (offsets)
+        that tie-back to original text.
+        This is done to bypass 512 token limit on transformer models by sequentially
+        feeding chunks of < 512 toks.
+        Example output:
+        [{...}, {"text": "...", 'start_idx': 31354, 'end_idx': 32648}, {...}]
+        """
+        wrds = text.replace('\n', ' ').split(" ")
+        resp = []
+        lst_chunk_idx = 0
+        i = 0
+
+        while True:
+            # words in the chunk and the overlapping portion
+            wrds_len = wrds[(length * i):(length * (i + 1))]
+            wrds_ovlp = wrds[(length * (i + 1)):((length * (i + 1)) + overlap)]
+            wrds_split = wrds_len + wrds_ovlp
+
+            # Break loop if no more words
+            if not wrds_split:
+                break
+
+            wrds_str = " ".join(wrds_split)
+            nxt_chunk_start_idx = len(" ".join(wrds_len))
+            lst_char_idx = len(" ".join(wrds_split))
+
+            resp_obj = {
+                "text": wrds_str,
+                "start_idx": lst_chunk_idx,
+                "end_idx": lst_char_idx + lst_chunk_idx,
+            }
+
+            resp.append(resp_obj)
+            lst_chunk_idx += nxt_chunk_start_idx + 1
+            i += 1
+        logging.info(f"Sliced transcript into {len(resp)} slices.")
+        return resp
+
+    @staticmethod
+    def combine_results(full_text: str, text_slices):
+        """
+        Given a full text and predictions of each slice combines predictions into a single text again.
+        Performs validataion wether text was combined correctly
+        """
+        split_full_text = full_text.replace('\n', ' ').split(" ")
+        split_full_text = [i for i in split_full_text if i]
+        split_full_text_len = len(split_full_text)
+        output_text = []
+        index = 0
+
+        if len(text_slices[-1]) <= 3 and len(text_slices) > 1:
+            text_slices = text_slices[:-1]
+
+        for _slice in text_slices:
+            slice_wrds = len(_slice)
+            for ix, wrd in enumerate(_slice):
+                # print(index, "|", str(list(wrd.keys())[0]), "|", split_full_text[index])
+                if index == split_full_text_len:
+                    break
+
+                if split_full_text[index] == str(list(wrd.keys())[0]) and \
+                        ix <= slice_wrds - 3 and text_slices[-1] != _slice:
+                    index += 1
+                    pred_item_tuple = list(wrd.items())[0]
+                    output_text.append(pred_item_tuple)
+                elif split_full_text[index] == str(list(wrd.keys())[0]) and text_slices[-1] == _slice:
+                    index += 1
+                    pred_item_tuple = list(wrd.items())[0]
+                    output_text.append(pred_item_tuple)
+        assert [i[0] for i in output_text] == split_full_text
+        return output_text
+
+    @staticmethod
+    def punctuate_texts(full_pred: list):
+        """
+        Given a list of Predictions from the model, applies the predictions to text,
+        thus punctuating it.
+        """
+        punct_resp = ""
+        for i in full_pred:
+            word, label = i
+            if label[-1] == "U":
+                punct_wrd = word.capitalize()
+            else:
+                punct_wrd = word
+
+            if label[0] != "O":
+                punct_wrd += label[0]
+
+            punct_resp += punct_wrd + " "
+        punct_resp = punct_resp.strip()
+        # Append trailing period if doesnt exist.
+        if punct_resp[-1].isalnum():
+            punct_resp += "."
+        return punct_resp
+
+
+if __name__ == "__main__":
+
+    start = time.time()
+    punct_model = RestorePuncts()
+
+    load_model = time.time()
+    print(f'Time to load model: {load_model - start}')
+    # read test file
+    # with open('en_lower.txt', 'r') as fp:
+    #     # test_sample = fp.read()
+    #     lines = fp.readlines()
+
+    with open('sample.vtt', 'r') as fp:
+        source_text = fp.read()
+
+    # captions = webvtt.read_buffer(StringIO(source_text))
+    captions = webvtt.read('sample.vtt')
+    source_sentences = [caption.text.replace('\r', '').replace('\n', ' ') for caption in captions]
+
+    # print(source_sentences)
+
+    sent = ' '.join(source_sentences)
+    punctuated = punct_model.punctuate(sent)
+
+    tokenised = sent_tokenize(punctuated)
+    # print(tokenised)
+
+    for i in range(len(tokenised)):
+        captions[i].text = tokenised[i]
+    # return captions.content
+    captions.save('my_captions.vtt')
+
+    end = time.time()
+    print(f'Time for run: {end - load_model}')
+    print(f'Total time: {end  - start}')

app.py

@@ -0,0 +1,29 @@
+import os
+import gradio as gr
+
+download="wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1-hzy09qi-OEogyge7rQG79K7iV4xsNWa' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\\n/p')&id=1-hzy09qi-OEogyge7rQG79K7iV4xsNWa\" -O indic-en.zip && rm -rf /tmp/cookies.txt"
+os.system(download)
+os.system('unzip /home/user/app/indic-en.zip')
+
+from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
+from inference.engine import Model
+indic2en_model = Model(expdir='/home/user/app/indic-en')
+
+INDIC = {"Assamese": "as", "Bengali": "bn", "Gujarati": "gu", "Hindi": "hi","Kannada": "kn","Malayalam": "ml", "Marathi": "mr", "Odia": "or","Punjabi": "pa","Tamil": "ta", "Telugu" : "te"}
+
+
+def translate(text, lang):
+  return indic2en_model.translate_paragraph(text, INDIC[lang], 'en')
+
+
+languages = list(INDIC.keys())
+drop_down = gr.inputs.Dropdown(languages, type="value", default="Hindi", label="Select Source Language")
+text = gr.inputs.Textbox(lines=5, placeholder="Enter Text to translate", default="", label="Enter Text in Source Language")
+text_ouptut = gr.outputs.Textbox(type="auto", label="Translated text in English")
+
+# example=[['I want to translate this sentence in Hindi','Hindi'],
+#         ['I am feeling very good today.', 'Bengali']]
+
+supported_lang = ', '.join(languages)
+iface = gr.Interface(fn=translate, inputs=[text,drop_down] , outputs=text_ouptut, title='IndicTrans NMT System', description = 'Currently the model supports ' + supported_lang, article = 'Original repository can be found [here](https://github.com/AI4Bharat/indicTrans)' , examples=None)
+iface.launch(enable_queue=True)

indic_nlp_library/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2013-present Anoop Kunchukuttan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

indic_nlp_library/README.md

@@ -0,0 +1,142 @@
+# Indic NLP Library
+
+The goal of the Indic NLP Library is to build Python based libraries for common text processing and Natural Language Processing in Indian languages. Indian languages share a lot of similarity in terms of script, phonology, language syntax, etc. and this library is an attempt to provide a general solution to very commonly required toolsets for Indian language text.
+
+The library provides the following functionalities:
+
+- Text Normalization
+- Script Information
+- Word Tokenization and Detokenization
+- Sentence Splitting 
+- Word Segmentation
+- Syllabification
+- Script Conversion
+- Romanization
+- Indicization
+- Transliteration
+- Translation
+
+The data resources required by the Indic NLP Library are hosted in a different repository. These resources are required for some modules. You can download from the [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources) project. 
+
+**If you are interested in Indian language NLP resources, you should check the [Indic NLP Catalog](https://github.com/indicnlpweb/indicnlp_catalog) for pointers.**
+
+## Pre-requisites
+
+- Python 3.x 
+   - (For Python 2.x version check the tag `PYTHON_2.7_FINAL_JAN_2019`. Not actively supporting Python 2.x anymore, but will try to maintain as much compatibility as possible)
+- [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources)
+- [Urduhack](https://github.com/urduhack/urduhack): Needed only if Urdu normalization is required. It has other dependencies like Tensorflow.
+- Other dependencies are listed in setup.py
+
+
+## Configuration
+
+- Installation from pip:
+
+    `pip install indic-nlp-library`
+
+- If you want to use the project from the github repo, add the project to the Python Path: 
+
+    - Clone this repository
+    - Install dependencies: `pip install -r requirements.txt`
+    - Run: `export PYTHONPATH=$PYTHONPATH:<project base directory>`
+
+- In either case, export the path to the _Indic NLP Resources_ directory
+
+    Run: `export INDIC_RESOURCES_PATH=<path to Indic NLP resources>` 
+
+## Usage 
+
+You can use the Python API to access all the features of the library. Many of the most common operations are also accessible via a unified commandline API. 
+
+### Getting Started
+
+Check [this IPython Notebook](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples.ipynb) for examples to use the Python API.
+  - You can find the Python 2.x Notebook [here](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples_2_7.ipynb) 
+
+### Documentation
+
+You can find detailed documentation  [HERE](https://indic-nlp-library.readthedocs.io/en/latest)
+
+This documents the Python API as well as the commandline reference. 
+
+## Citing
+
+If you use this library, please include the following citation: 
+
+```
+@misc{kunchukuttan2020indicnlp,
+author = "Anoop Kunchukuttan",
+title = "{The IndicNLP Library}",
+year = "2020",
+howpublished={\url{https://github.com/anoopkunchukuttan/indic_nlp_library/blob/master/docs/indicnlp.pdf}}
+}
+```
+You can find the document [HERE](docs/indicnlp.pdf)
+
+## Website
+
+`http://anoopkunchukuttan.github.io/indic_nlp_library`
+
+## Author
+Anoop Kunchukuttan ([[email protected]]([email protected]))
+
+## Companies, Organizations, Projects using IndicNLP Library
+
+- [AI4Bharat-IndicNLPSuite](https://indicnlp.ai4bharat.org)
+- [The Classical Language Toolkit](http://cltk.org)
+- [Microsoft NLP Recipes](https://github.com/microsoft/nlp-recipes)
+- [Facebook M2M-100](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100)
+
+## Revision Log
+
+
+0.81 : 26 May 2021 
+    
+    - Bug fix in version number extraction
+
+0.80 : 24 May 2021 
+
+    - Improved sentence splitting
+    - Bug fixes
+    - Support for Urdu Normalizer
+
+0.71 : 03 Sep 2020 
+
+    - Improved documentation
+    - Bug fixes
+
+0.7 : 02 Apr 2020:
+
+    - Unified commandline 
+    - Improved documentation
+    - Added setup.py
+
+0.6 : 16 Dec 2019:
+
+    - New romanizer and indicizer
+    - Script Unifiers
+    - Improved script normalizers
+    - Added contrib directory for sample uses
+    - changed to MIT license 
+
+0.5 : 03 Jun 2019: 
+
+    - Improved word tokenizer to handle dates and numbers. 
+    - Added sentence splitter that can handle common prefixes/honorofics and uses some heuristics.
+    - Added detokenizer
+    - Added acronym transliterator that can convert English acronyms to Brahmi-derived scripts
+
+0.4 : 28 Jan 2019: Ported to Python 3, and lots of feature additions since last release; primarily around script information, script similarity and syllabification.
+
+0.3 : 21 Oct 2014: Supports morph-analysis between Indian languages
+
+0.2 : 13 Jun 2014: Supports transliteration between Indian languages and tokenization of Indian languages 
+
+0.1 : 12 Mar 2014: Initial version. Supports text normalization.
+
+## LICENSE
+
+Indic NLP Library is released under the MIT license
+
+

indic_nlp_library/contrib/README.md

@@ -0,0 +1,7 @@
+# Contrib
+
+Contains additional utilities and applications using Indic NLP library core
+
+- `indic_scraper_project_sample.ipynb`: A simple pipeline for building monolingual corpora for Indian languages from crawled web content, Wikipedia, etc. An extensible framework which allows incorporation of website specific extractors, whereas generic NLP tasks like tokenization, sentence splitting, normalization, etc. are handled by the framework.
+- `correct_moses_tokenizer.py`: This script corrects the incorrect tokenization done by Moses tokenizer.  The Moses tokenizer splits on nukta and halant characters.
+- `hindi_to_kannada_transliterator.py`: This script transliterates Hindi to Kannada. It removes/remaps characters only found in Hindi. It also adds halanta to words ending with consonant - as is the convention in Kannada.

indic_nlp_library/contrib/correct_moses_tokenizer.py

@@ -0,0 +1,29 @@
+import sys
+from indicnlp import langinfo
+from indicnlp import loader 
+
+if __name__ == '__main__':
+    """
+        This script corrects the incorrect tokenization done by Moses tokenizer.
+        The Moses tokenizer splits on nukta and halant characters
+        Usage: python correct_moses_tokenizer.py <infname> <outfname> <langcode>
+    """
+
+    loader.load()
+
+    infname=sys.argv[1]
+    outfname=sys.argv[2]
+    lang=sys.argv[3]
+
+    halant_char=langinfo.offset_to_char(langinfo.HALANTA_OFFSET,lang)
+    nukta_char=langinfo.offset_to_char(langinfo.NUKTA_OFFSET,lang)
+
+    with open(infname,'r',encoding='utf-8') as infile, \
+         open(outfname,'w',encoding='utf-8') as outfile:
+        for line in infile:
+            outfile.write(
+                    line.replace(
+                        ' {} '.format(halant_char), halant_char).replace(
+                        ' {} '.format(nukta_char), nukta_char).replace(
+                        ' {}{}'.format(nukta_char,halant_char),'{}{}'.format(nukta_char,halant_char))    
+                    )

indic_nlp_library/contrib/hindi_to_kannada_transliterator.py

@@ -0,0 +1,62 @@
+import sys
+from indicnlp import common
+common.set_resources_path(INDIC_NLP_RESOURCES)
+
+from indicnlp import loader
+from indicnlp.normalize import indic_normalize
+from indicnlp.transliterate import unicode_transliterate
+
+if __name__ == '__main__': 
+    """
+    This script transliterates Hindi to Kannada. It removes/remaps 
+    characters only found in Hindi. It also adds halanta to words ending
+    with consonant - as is the convention in Kannada
+    """
+
+    infname=sys.argv[1]  # one sentence/word per line. Sentences should be space-tokenized
+    outfname=sys.agv[2]
+    loader.load()
+
+    normalizer_factory=indic_normalize.IndicNormalizerFactory()
+    normalizer=normalizer_factory.get_normalizer('hi')
+
+    with open(infname,'r',encoding='utf-8') as infile, \
+         open(outfname,'w',encoding='utf-8') as outfile:
+        for line in infile: 
+            line=line.strip()
+            line=normalizer.normalize(line)
+    
+            ## replace chandrabindus with anusvara
+            line=line.replace('\u0900','\u0902')
+            line=line.replace('\u0901','\u0902')
+    
+            ### replace chandra e and o diacritics with e and o respectively
+            #line=line.replace('\u0945','\u0947')
+            #line=line.replace('\u0949','\u094b')
+   
+            ### replace chandra e and o diacritics with a diacritic
+            ## this seems to be general usage
+            line=line.replace('\u0945','\u093e')
+            line=line.replace('\u0949','\u093e')
+   
+            ## remove nukta 
+            line=line.replace('\u093c','')
+
+            ## add halant if word ends with consonant
+            #if isc.is_consonant(isc.get_phonetic_feature_vector(line[-1],'hi')):
+            #    line=line+'\u094d'
+            words=line.split(' ')
+            outwords=[]
+            for word in line.split(' '):
+                if isc.is_consonant(isc.get_phonetic_feature_vector(word[-1],'hi')):
+                    word=word+'\u094d'
+            outwords.append(word)
+            line=' '.join(outwords)
+
+    
+            ## script conversion 
+            line=unicode_transliterate.UnicodeIndicTransliterator.transliterate(line,'hi','kn')
+            
+            outfile.write(line+'\n')
+
+

indic_nlp_library/contrib/indic_scraper_project_sample.ipynb

@@ -0,0 +1,569 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Pre-requisites\n",
+    "\n",
+    "- Python 3.5+\n",
+    "- Python packages: \n",
+    "    - `pip install bs4 pandas mmh3`\n",
+    "- [Indic NLP Library](https://github.com/anoopkunchukuttan/indic_nlp_library)\n",
+    "- [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Initialize the Indic NLP Library\n",
+    "\n",
+    "Run the cell below to initialize the Indic NLP Library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The path to the local git repo for Indic NLP Library\n",
+    "INDIC_NLP_LIB_HOME=\"/disk1/src/indic_nlp_library\"\n",
+    "\n",
+    "# The path to the local git repo for Indic NLP Resources\n",
+    "INDIC_NLP_RESOURCES=\"/disk1/src/indic_nlp_resources\"\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('{}/src'.format(INDIC_NLP_LIB_HOME))\n",
+    "\n",
+    "from indicnlp import common\n",
+    "common.set_resources_path(INDIC_NLP_RESOURCES)\n",
+    "\n",
+    "from indicnlp import loader\n",
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from bs4 import BeautifulSoup\n",
+    "import os\n",
+    "import string\n",
+    "import indicnlp\n",
+    "from indicnlp.tokenize import indic_tokenize\n",
+    "from indicnlp.normalize import indic_normalize\n",
+    "from indicnlp.transliterate import unicode_transliterate\n",
+    "from indicnlp.tokenize import sentence_tokenize\n",
+    "import re\n",
+    "import collections\n",
+    "import random\n",
+    "import mmh3"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Common Functions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def preprocess_sent(text,lang,normalizer):\n",
+    "    \"\"\"\n",
+    "        Pre-process text (normalization and tokenization)\n",
+    "        \n",
+    "        text: text string to preprocess\n",
+    "        lang: language code (2-letter ISO code)\n",
+    "        normalizer: normalizer object for language\n",
+    "        \n",
+    "        returns the processed text string\n",
+    "    \"\"\"\n",
+    "    return ' '.join(indic_tokenize.trivial_tokenize(normalizer.normalize(text.replace('\\n',' ')),lang)) \n",
+    "\n",
+    "def sent_split(text,lang):\n",
+    "    \"\"\"\n",
+    "        Sentence splitter\n",
+    "        \n",
+    "        text: text to sentence split \n",
+    "        lang: language\n",
+    "        \n",
+    "        returns list of sentences \n",
+    "    \"\"\"\n",
+    "    return  sentence_tokenize.sentence_split(text,lang)\n",
+    "\n",
+    "def extract_all_content(indir,lang,\n",
+    "                        article_extract_fn,\n",
+    "                        preprocess_fn=preprocess_sent,\n",
+    "                        narticles=-1,\n",
+    "                        start_artid=0):\n",
+    "    \"\"\"\n",
+    "    This method reads all files from the input directory, extracts text content from each file,\n",
+    "    and pre-processes the text. This method is a generator. \n",
+    "    For each sentence, the method yields a tuple of the format: \n",
+    "    \n",
+    "    (artid, fname, paraid, sentid, processed_text)\n",
+    "    \n",
+    "    indir: path to input directoryo containing files to be parsed \n",
+    "    \n",
+    "    lang: language to the files in the input directory\n",
+    "    \n",
+    "    article_extract_fn: the function to extract text content from each file. \n",
+    "        Signature of the function: get_article_contents(fname,lang,encoding) \n",
+    "          `fname` is name of the file, `lang` is langcode, \n",
+    "          `encoding` is text-encoding (default=utf-8). \n",
+    "        The function yields a tuple (paraid, sentid, extracted_text) \n",
+    "        for each  sentence.\n",
+    "        \n",
+    "    preprocess_fn: pre-processing function to apply to the extracted text. \n",
+    "        The function takes a string as input and returns processed string as output.\n",
+    "        \n",
+    "    narticles: extract and process the first `narticles` from input directory. \n",
+    "        if narticles=-1 (default), all files are extracted\n",
+    "        \n",
+    "    start_artid: the start of the article id to assign to extracted articles (default=0)\n",
+    "    \n",
+    "    \"\"\"\n",
+    "\n",
+    "    fnames = os.listdir(indir)\n",
+    "    if narticles>0:\n",
+    "        fnames=fnames[:narticles]\n",
+    "    nsent=0\n",
+    "\n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)\n",
+    "             \n",
+    "    print('Number of articles: {}'.format(len(fnames)))\n",
+    "    for artid, fname in enumerate(fnames,start_artid):\n",
+    "#         print(fname)\n",
+    "        if artid%100 == 0:\n",
+    "            print('({}|{})'.format(artid,nsent),end=' ... ')\n",
+    "        \n",
+    "        try:\n",
+    "            fpath=os.sep.join([indir,fname])\n",
+    "            for paraid, sentid, sent in article_extract_fn(fpath,lang):\n",
+    "                nsent+=1\n",
+    "                yield( ( artid, fname, paraid, sentid, preprocess_fn(sent,lang,normalizer) ) )\n",
+    "        except:\n",
+    "            print('Cannot parse {}'.format(fname))\n",
+    "                \n",
+    "def write_corpus(corpus_iterator,content_fname,article_mapping_fname,delimiter=' ||| ', encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Writes the extracted corpus to a file. The extracted data is organized in terms of articles, paragraphs \n",
+    "    and sentences. The following is the format of the output file: \n",
+    "        - one line per sentence\n",
+    "        - format of line: article_id, para_id, sent_id, sentence\n",
+    "    In addition to the content file mention, a metadata file which maps the article id to the filename is also written. \n",
+    "    \n",
+    "    corpus_iterator: iterator over the corpus, yielding tuple (artid, fname, paraid, sentid, processed_text). \n",
+    "        The function `extract_all_content` yields a generator in this format. \n",
+    "    content_fname: output content file to write the extracted data to in the format mentioned above\n",
+    "    article_mapping_fname: output metadata file to write article id to filename mapping.\n",
+    "    delimiter=' ||| ': delimiter for the content file. The default delimiter is the same \n",
+    "                        as used in the Moses phrase table\n",
+    "    encoding: text encoding default - 'utf-8'\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    \n",
+    "    artid_name_mapping={}\n",
+    "    with open(content_fname,'w',encoding=encoding) as contentfile:\n",
+    "        for artid, fname, paraid, sentid, text in corpus_iterator:\n",
+    "            contentfile.write(delimiter.join([str(artid), str(paraid), str(sentid), text]) + '\\n')\n",
+    "            artid_name_mapping[artid]=fname\n",
+    "\n",
+    "    with open(article_mapping_fname,'w',encoding=encoding) as artmappingfile:\n",
+    "        for artid, name in sorted(artid_name_mapping.items(),key=lambda x: x[0]):\n",
+    "            artmappingfile.write('{} {} {}\\n'.format(artid,delimiter,name))\n",
+    "\n",
+    "def convert_txt_to_csv_format(infname, outfname, encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    convert txt file to csv format. This method is used when the text file is directly available.\n",
+    "    The input file has one sentence per line. Assumed to be preprocessed (tokenized, normalized)\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for i, line in enumerate(infile):\n",
+    "            outfile.write('0 ||| 0 ||| {} ||| {}\\n'.format(i,line.strip()))\n",
+    "            \n",
+    "def preprocess_convert_txt_to_csv_format(infname, outfname, lang, encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Convert raw text file to csv format\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)\n",
+    "    \n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        i=0\n",
+    "        for line in infile:\n",
+    "            sents = sent_split(line.strip(),lang)\n",
+    "            for sent in sents:\n",
+    "                outfile.write('0 ||| 0 ||| {} ||| {}\\n'.format(i,\n",
+    "                                                    preprocess_sent(sent.strip(), lang, normalizer)) )\n",
+    "                i=i+1\n",
+    "\n",
+    "def print_txt(infnames, outfname, encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Extract only the text from the content csv file. The output file has one sentence per file.\n",
+    "    \"\"\"\n",
+    "    with open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for infname in filter(lambda x: os.path.isfile(x),infnames):\n",
+    "            with open(infname,'r',encoding=encoding) as infile:\n",
+    "                for i, line in enumerate(infile):\n",
+    "                    fields=line.strip().split('|||')\n",
+    "                    if len(fields) >=4:\n",
+    "                        outfile.write('{}\\n'.format(fields[3].strip()))\n",
+    "                        \n",
+    "# def dedup_and_print_txt(infnames, outfname, encoding='utf-8'):\n",
+    "    \n",
+    "#     total=0\n",
+    "#     unique=0\n",
+    "#     hash_codes=set()\n",
+    "    \n",
+    "#     with open(outfname,'w',encoding=encoding) as outfile: \n",
+    "#         for infname in filter(lambda x: os.path.isfile(x),infnames):\n",
+    "#             with open(infname,'r',encoding=encoding) as infile:\n",
+    "#                 for i, line in enumerate(infile):\n",
+    "#                     fields=line.strip().split('|||')\n",
+    "#                     if len(fields) >=4:\n",
+    "#                         sent=fields[3].strip()\n",
+    "#                         total+=1\n",
+    "#                         hs=hash(sent)\n",
+    "#                         if hs not in hash_codes:\n",
+    "#                             outfile.write('{}\\n'.format(sent))\n",
+    "#                             hash_codes.add(hs)\n",
+    "#                             unique+=1\n",
+    "    \n",
+    "#     print('Total: {}'.format(total))\n",
+    "#     print('Unique: {}'.format(unique))\n",
+    "\n",
+    "def dedup_shuffle_and_print_txt(infnames, outfname, max_buf_size=100000,encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    The method creates a sentence level corpora from multiple content csv files.\n",
+    "    All sentences are extracted, they are de-duplicated using murmurhash and shuffled\n",
+    "    before writing the entire corpus to the output file. The output file has one sentence per line.\n",
+    "\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    total=0\n",
+    "    unique=0\n",
+    "    hash_codes=set()\n",
+    "    sent_buffer=[]\n",
+    "    \n",
+    "    with open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for infname in filter(lambda x: os.path.isfile(x),infnames):\n",
+    "            print('Processing: {}'.format(infname))\n",
+    "            with open(infname,'r',encoding=encoding) as infile:\n",
+    "                for i, line in enumerate(infile):\n",
+    "                    fields=line.strip().split('|||')\n",
+    "                    if len(fields) >=4:\n",
+    "                        sent=fields[3].strip()\n",
+    "                        total+=1\n",
+    "#                         hs=hash(sent)\n",
+    "                        hs=mmh3.hash128(sent)\n",
+    "                        if hs not in hash_codes:\n",
+    "#                             outfile.write('{}\\n'.format(sent))\n",
+    "                            sent_buffer.append(sent)\n",
+    "                            hash_codes.add(hs)\n",
+    "                            unique+=1\n",
+    "                    if len(sent_buffer)>=max_buf_size:\n",
+    "                        random.shuffle(sent_buffer)\n",
+    "                        for sent in sent_buffer: \n",
+    "                            outfile.write('{}\\n'.format(sent))\n",
+    "                        sent_buffer.clear()\n",
+    "                \n",
+    "        if len(sent_buffer)>0:\n",
+    "            random.shuffle(sent_buffer)\n",
+    "            for sent in sent_buffer: \n",
+    "                outfile.write('{}\\n'.format(sent))\n",
+    "            sent_buffer.clear()                    \n",
+    "                        \n",
+    "    print('Total: {}'.format(total))\n",
+    "    print('Unique: {}'.format(unique))\n",
+    "\n",
+    "def extract_wikiextractor_file(infname, outfname, lang, \n",
+    "                               encoding='utf-8', delimiter=' ||| ', preprocess_fn=preprocess_sent):\n",
+    "    \"\"\"\n",
+    "    Extract text content into a content csv file from wikipedia article page. \n",
+    "    The wikipedia article page is the output from `wikiextractor` [https://github.com/attardi/wikiextractor] \n",
+    "    \n",
+    "    \"\"\"\n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)\n",
+    "    \n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        artid=-1\n",
+    "        paraid=0\n",
+    "        for line in infile:\n",
+    "            if line.find('<doc')==0:\n",
+    "                artid+=1\n",
+    "                paraid=0\n",
+    "                continue\n",
+    "            if line.find('</doc')==0:\n",
+    "                continue\n",
+    "            if len(line.strip())>0:\n",
+    "                for sentid, sent in enumerate(sent_split(line.strip(),lang)):\n",
+    "                    sent=sent.strip()\n",
+    "                    if sent!='':\n",
+    "                        sent = preprocess_fn(sent,lang,normalizer)\n",
+    "                        outfile.write(delimiter.join([str(artid), str(paraid), str(sentid), sent]) + '\\n')\n",
+    "                paraid+=1\n",
+    "\n",
+    "                \n",
+    "def extract_leipzig_corpus(infname,outfname,lang,encoding='utf-8'):\n",
+    "    \"\"\"\n",
+    "    Extractor for files form the Leipzig corpus\n",
+    "    [http://wortschatz.uni-leipzig.de/en/download/]\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    normalizer_factory=indic_normalize.IndicNormalizerFactory()\n",
+    "    normalizer=normalizer_factory.get_normalizer(lang)    \n",
+    "\n",
+    "    with open(infname,'r',encoding=encoding) as infile, \\\n",
+    "         open(outfname,'w',encoding=encoding) as outfile: \n",
+    "        for i, line in enumerate(infile):\n",
+    "            outfile.write('0 ||| 0 ||| {} ||| {}\\n'.format(i,preprocess_sent(line,lang,normalizer)))                \n",
+    "                \n",
+    "def dataset_stats(fname):\n",
+    "    \"\"\"\n",
+    "    Extracts dataset statistics from the final extracted file. This input file contains\n",
+    "    one sentence per line. The sentences are tokenized.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    all_puncs=set(string.punctuation+'\\u0964\\u0965')\n",
+    "    \n",
+    "    sent_count=0\n",
+    "    token_cnt=0\n",
+    "    true_token_cnt=0\n",
+    "    tokens=set()\n",
+    "    \n",
+    "    with open(fname,'r',encoding='utf-8') as infile:\n",
+    "        for line in infile:\n",
+    "            sent_count+=1\n",
+    "            a=line.strip().split(' ')\n",
+    "            token_cnt+=len(a)\n",
+    "            b=list(filter(lambda x: x not in all_puncs,a))\n",
+    "            true_token_cnt+=len(b)\n",
+    "            tokens.update(b)\n",
+    "    \n",
+    "    print('== Stats ==')\n",
+    "    print('Sent count: {}'.format(sent_count))\n",
+    "    print('Token count: {}'.format(token_cnt))\n",
+    "    print('True Token count: {}'.format(true_token_cnt))\n",
+    "    print('Unique Token count: {}'.format(len(tokens)))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Marathi"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Wikipedia"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Wikipedia extraction commands using wikiextractor\n",
+    "\n",
+    "```\n",
+    "### This uses WikiExtractor (https://github.com/attardi/wikiextractor)\n",
+    "\n",
+    "x=/disk1/crawl_project/ta/wikipedia\n",
+    "mkdir $x\n",
+    "cd $x\n",
+    "wget https://dumps.wikimedia.org/tawiki/20190501/tawiki-20190501-pages-articles-multistream.xml.bz2\n",
+    "cd /disk1/src/wikiextractor\n",
+    "python3 WikiExtractor.py -cb 250k -o $x/extracted $x/tawiki-20190501-pages-articles-multistream.xml.bz2\n",
+    "cd -\n",
+    "find extracted -name '*bz2' -exec bunzip2 -c {} \\; > text.xml\n",
+    "rm text.xml\n",
+    "rm tawiki-20190501-pages-articles-multistream.xml.bz2\n",
+    "rm -rf extracted\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "mrwiki-20190401-pages-articles-multistream.xml.bz2\n",
+    "\n",
+    "INFO: Finished 1-process extraction of 53715 articles in 123.6s (434.7 art/s)\n",
+    "\n",
+    "INFO: total of page: 102025, total of articl page: 53715; total of used articl page: 53715"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Post-processing output generated by wikiextractor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## tex.xml is extracted as shown in commanfs above\n",
+    "extract_wikiextractor_file('text.xml',\n",
+    "                           'content_fname1.csv',\n",
+    "                           'mr')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Loksatta"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Extractor function for Marathi Loksatta page**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_article_contents_mr_loksatta(fname,lang,encoding='utf-8'):\n",
+    "    with open(fname,'r',encoding=encoding) as infile: \n",
+    "        soup = BeautifulSoup(infile)\n",
+    "        for elem in soup.find_all('div'):\n",
+    "            if 'itemprop' in elem.attrs and 'articleBody' in elem['itemprop']:\n",
+    "                filtered_paras=list(filter(lambda x: x.name=='p' and len(x.attrs)==0,elem.children))\n",
+    "                paraid=0\n",
+    "                for blockid, block in enumerate(filtered_paras):\n",
+    "#                     print('Para: {}'.format(blockid))\n",
+    "#                     print(list(block.strings))\n",
+    "                    text=' '.join(block.strings)\n",
+    "                    if blockid==0 and text.find(':')>=0 and text.find(':')<20:\n",
+    "                        text=':'.join(text.split(':')[1:])\n",
+    "                    for para_text in text.split('\\n'): \n",
+    "                        for sentid, sent in enumerate(sent_split(para_text,lang)):\n",
+    "                            sent=sent.strip()\n",
+    "                            if sent!='':\n",
+    "    #                             print('{}: {}'.format(sentid, sent))\n",
+    "                                yield((paraid,sentid,sent))\n",
+    "    #                             yield((paraid,sentid,preprocess_sent(sent,'ml',normalizer)))\n",
+    "    #                     print()   \n",
+    "                        paraid+=1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Extracting data from crawled HTML files**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lang='mr'\n",
+    "posts_dir='directory_containing_crawled_html_pages'\n",
+    "content_fname='content_fname2.csv'\n",
+    "article_mapping_fname='article_mapping_fname'\n",
+    "get_article_contents=get_article_contents_mr_loksatta\n",
+    "narticles=-1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "write_corpus(\n",
+    "             extract_all_content(posts_dir, lang, article_extract_fn=get_article_contents,narticles=narticles),\n",
+    "             content_fname,\n",
+    "             article_mapping_fname\n",
+    "            )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Aggregating all crawled data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### aggregating, de-duplicating and shuffling all the data \n",
+    "dedup_shuffle_and_print_txt([ 'content_fname1.csv', 'content_fname2.csv'  ],  'output_fname.txt' )\n",
+    "### extract dataset statistics\n",
+    "dataset_stats('output_fname.txt')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {
+    "height": "703px",
+    "width": "326px"
+   },
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

indic_nlp_library/docs/Makefile

@@ -0,0 +1,153 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/IndicNLPLibrary.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/IndicNLPLibrary.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/IndicNLPLibrary"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/IndicNLPLibrary"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

indic_nlp_library/docs/cmd.rst

@@ -0,0 +1,8 @@
+Commandline
+===========
+
+.. argparse::
+   :module: indicnlp.cli.cliparser
+   :func: get_parser
+   :prog: cliparser.py
+

indic_nlp_library/docs/code.rst

@@ -0,0 +1,5 @@
+Auto Generated Documentation
+============================
+
+.. automodule:: indicnlp.langinfo indicnlp.common
+   :members: 

indic_nlp_library/docs/conf.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# Indic NLP Library documentation build configuration file, created by
+# sphinx-quickstart on Tue Nov  3 01:50:37 2015.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'sphinxarg.ext']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Indic NLP Library'
+copyright = '2015, Anoop Kunchukuttan'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.2'
+# The full version, including alpha/beta/rc tags.
+release = '0.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'IndicNLPLibrarydoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'IndicNLPLibrary.tex', 'Indic NLP Library Documentation',
+   'Anoop Kunchukuttan', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'indicnlplibrary', 'Indic NLP Library Documentation',
+     ['Anoop Kunchukuttan'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'IndicNLPLibrary', 'Indic NLP Library Documentation',
+   'Anoop Kunchukuttan', 'IndicNLPLibrary', 'NLP library for Indian languages',
+   'NLP'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'

indic_nlp_library/docs/index.rst

@@ -0,0 +1,22 @@
+.. Indic NLP Library documentation master file, created by
+   sphinx-quickstart on Tue Nov  3 01:50:37 2015.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+:github_url: https://github.com/anoopkunchukuttan/indic_nlp_library
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Packages
+
+  indicnlp
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Commandline
+
+  cmd
+
+
+
+

indic_nlp_library/docs/indicnlp.MD

@@ -0,0 +1,122 @@
+# Indic NLP Library
+##  A unified approach to NLP for Indian languages
+
+### Anoop Kunchukuttan (`[email protected]`)
+
+The goal of the Indic NLP Library is to build Python based libraries for common text processing and Natural Language Processing in Indian languages. Indian languages share a lot of similarity in terms of script, phonology, language syntax, etc. and this library is an attempt to provide a general solution to very commonly required toolsets for Indian language text.
+
+The library provides the following functionalities:
+
+- Text Normalization
+- Script Information
+- Word Tokenization and Detokenization
+- Sentence Splitting 
+- Word Segmentation
+- Syllabification
+- Script Conversion
+- Romanization
+- Indicization
+- Transliteration
+- Translation
+
+The data resources required by the Indic NLP Library are hosted in a different repository. These resources are required for some modules. You can download from the [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources) project. 
+
+**If you are interested in Indian language NLP resources, you should check the [Indic NLP Catalog](https://github.com/anoopkunchukuttan/indic_nlp_library) for pointers.**
+
+## Pre-requisites
+
+- Python 3.x 
+   - (For Python 2.x version check the tag `PYTHON_2.7_FINAL_JAN_2019`. Not actively supporting Python 2.x anymore, but will try to maintain as much compatibility as possible)
+- [Indic NLP Resources](https://github.com/anoopkunchukuttan/indic_nlp_resources)
+- Other dependencies are listed in setup.py
+
+
+## Configuration
+
+- Installation from pip:
+
+    `pip install indic-nlp-library`
+
+- If you want to use the project from the github repo, add the project to the Python Path: 
+
+    - Clone this repository
+    - Install dependencies: `pip install -r requirements.txt`
+    - Run: `export PYTHONPATH=$PYTHONPATH:<project base directory>`
+
+- In either case, export the path to the _Indic NLP Resources_ directory
+
+    Run: `export INDIC_RESOURCES_PATH=<path to Indic NLP resources>` 
+
+## Usage 
+
+You can use the Python API to access all the features of the library. Many of the most common operations are also accessible via a unified commandline API. 
+
+### Getting Started
+
+Check [this IPython Notebook](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples.ipynb) for examples to use the Python API.
+  - You can find the Python 2.x Notebook [here](http://nbviewer.ipython.org/url/anoopkunchukuttan.github.io/indic_nlp_library/doc/indic_nlp_examples_2_7.ipynb) 
+
+### Documentation
+
+You can find detailed documentation  [HERE](https://indic-nlp-library.readthedocs.io/en/latest)
+
+This documents the Python API as well as the commandline reference. 
+
+## Citing
+
+If you use this library, please include the following citation: 
+
+```
+@unpublished{kunchukuttan2020indicnlp,
+author = "Anoop Kunchukuttan",
+title = "The IndicNLP Library",
+year = "2020",
+}
+```
+You can find the document [HERE](docs/indicnlp.pdf)
+
+## Website
+
+`http://anoopkunchukuttan.github.io/indic_nlp_library`
+
+## Author
+Anoop Kunchukuttan ([[email protected]]([email protected]))
+
+## Version: 0.7
+
+## Revision Log
+
+0.7 : 02 Apr 2020:
+
+    - Unified commandline 
+    - Improved documentation
+    - Added setup.py
+
+0.6 : 16 Dec 2019:
+
+    - New romanizer and indicizer
+    - Script Unifiers
+    - Improved script normalizers
+    - Added contrib directory for sample uses
+    - changed to MIT license 
+
+0.5 : 03 Jun 2019: 
+
+    - Improved word tokenizer to handle dates and numbers. 
+    - Added sentence splitter that can handle common prefixes/honorofics and uses some heuristics.
+    - Added detokenizer
+    - Added acronym transliterator that can convert English acronyms to Brahmi-derived scripts
+
+0.4 : 28 Jan 2019: Ported to Python 3, and lots of feature additions since last release; primarily around script information, script similarity and syllabification.
+
+0.3 : 21 Oct 2014: Supports morph-analysis between Indian languages
+
+0.2 : 13 Jun 2014: Supports transliteration between Indian languages and tokenization of Indian languages 
+
+0.1 : 12 Mar 2014: Initial version. Supports text normalization.
+
+## LICENSE
+
+Indic NLP Library is released under the MIT license
+
+

indic_nlp_library/docs/indicnlp.cli.rst

@@ -0,0 +1,11 @@
+cli Package
+=============
+
+:mod:`cliparser` Module
+--------------------------------
+
+.. automodule:: indicnlp.cli.cliparser
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

indic_nlp_library/docs/indicnlp.morph.rst

@@ -0,0 +1,11 @@
+morph Package
+=============
+
+:mod:`unsupervised_morph` Module
+--------------------------------
+
+.. automodule:: indicnlp.morph.unsupervised_morph
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

indic_nlp_library/docs/indicnlp.normalize.rst

@@ -0,0 +1,15 @@
+normalize Package
+=================
+
+:mod:`indic_normalize` Module
+-----------------------------
+
+.. automodule:: indicnlp.normalize.indic_normalize
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autoclass:: indicnlp.normalize.indic_normalize.
+    :members:
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/indicnlp.rst

@@ -0,0 +1,47 @@
+indicnlp Package
+================
+
+:mod:`common` Module
+--------------------
+
+.. automodule:: indicnlp.common
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`langinfo` Module
+----------------------
+
+.. automodule:: indicnlp.langinfo
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`loader` Module
+--------------------
+
+.. automodule:: indicnlp.loader
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+    indicnlp.cli
+    indicnlp.morph
+    indicnlp.normalize
+    indicnlp.script
+    indicnlp.syllable
+    indicnlp.tokenize
+    indicnlp.transliterate
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

indic_nlp_library/docs/indicnlp.script.rst

@@ -0,0 +1,26 @@
+script Package
+==============
+
+:mod:`indic_scripts` Module
+---------------------------
+
+.. automodule:: indicnlp.script.indic_scripts
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`english_script` Module
+---------------------------
+
+.. automodule:: indicnlp.script.english_script
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`phonetic_sim` Module
+---------------------------
+
+.. automodule:: indicnlp.script.phonetic_sim
+    :members: 
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/indicnlp.syllable.rst

@@ -0,0 +1,11 @@
+syllable Package
+==============
+
+:mod:`syllabifier` Module
+---------------------------
+
+.. automodule:: indicnlp.syllable.syllabifier
+    :members:
+    :undoc-members:
+    :show-inheritance:
+

indic_nlp_library/docs/indicnlp.tokenize.rst

@@ -0,0 +1,26 @@
+tokenize Package
+================
+
+:mod:`indic_tokenize` Module
+----------------------------
+
+.. automodule:: indicnlp.tokenize.indic_tokenize
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`indic_detokenize` Module
+------------------------------
+
+.. automodule:: indicnlp.tokenize.indic_detokenize
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`sentence_tokenize` Module
+----------------------------
+
+.. automodule:: indicnlp.tokenize.sentence_tokenize
+    :members:
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/indicnlp.transliterate.rst

@@ -0,0 +1,34 @@
+transliterate Package
+=====================
+
+:mod:`sinhala_transliterator` Module
+------------------------------------
+
+.. automodule:: indicnlp.transliterate.sinhala_transliterator
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`unicode_transliterate` Module
+-----------------------------------
+
+.. automodule:: indicnlp.transliterate.unicode_transliterate
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`acronym_transliterator` Module
+-----------------------------------
+
+.. automodule:: indicnlp.transliterate.acronym_transliterator
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`script_unifier` Module
+-----------------------------------
+
+.. automodule:: indicnlp.transliterate.script_unifier
+    :members:
+    :undoc-members:
+    :show-inheritance:

indic_nlp_library/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

indic_nlp_library/docs/modules.rst

@@ -0,0 +1,7 @@
+indicnlp
+===
+
+.. toctree::
+   :maxdepth: 4
+
+   indicnlp

indic_nlp_library/indicnlp/__init__.py

@@ -0,0 +1,10 @@
+import os
+import sys
+
+try:
+    from .version import __version__  # noqa
+except ImportError:
+    version_txt = os.path.join(os.path.dirname(__file__), "version.txt")
+    with open(version_txt) as f:
+        __version__ = f.read().strip()
+

indic_nlp_library/indicnlp/cli/cliparser.py

@@ -0,0 +1,266 @@
+import argparse 
+import sys
+
+from indicnlp import loader
+from indicnlp.tokenize import indic_tokenize
+from indicnlp.tokenize import indic_detokenize
+from indicnlp.normalize import indic_normalize
+from indicnlp.morph import unsupervised_morph
+from indicnlp.tokenize import sentence_tokenize
+from indicnlp.syllable import  syllabifier
+from indicnlp.transliterate import unicode_transliterate
+from indicnlp.transliterate import script_unifier
+
+DEFAULT_ENCODING='utf-8'
+
+def run_detokenize(args):
+    for line in args.infile:
+        args.outfile.write(indic_detokenize.trivial_detokenize(line,args.lang))
+
+def run_tokenize(args):
+    for line in args.infile:
+        args.outfile.write(' '.join(
+            indic_tokenize.trivial_tokenize(line,args.lang)))
+
+def run_sentence_split(args):            
+    text=' '.join([  l.replace('\n','').replace('\r','') for l in args.infile])
+    outlines=sentence_tokenize.sentence_split(text,args.lang)
+    for line in outlines:
+        args.outfile.write(line+'\n')
+
+def run_normalize(args):
+
+    # TODO: add more options to cli
+    remove_nuktas=False
+    normalize_nasals='do_nothing'
+
+    # create normalizer
+    factory=indic_normalize.IndicNormalizerFactory()
+    normalizer=factory.get_normalizer(args.lang,
+            remove_nuktas=remove_nuktas,
+            nasals_mode=normalize_nasals)
+
+    # DO normalization 
+    for line in args.infile:
+        normalized_line=normalizer.normalize(line)
+        args.outfile.write(normalized_line)
+
+def run_morph(args):
+
+    add_marker=False
+    analyzer=unsupervised_morph.UnsupervisedMorphAnalyzer(args.lang,add_marker)
+    for line in args.infile:
+        morph_tokens=analyzer.morph_analyze_document(line.strip().split(' '))
+        args.outfile.write(' '.join(morph_tokens) + '\n')
+
+def run_syllabify(args):
+    for line in args.infile:
+        new_line = ' '.join(
+                        [ ' '.join(syllabifier.orthographic_syllabify(w,args.lang)) 
+                                for w in line.strip().split(' ')  ]
+                    )
+        args.outfile.write(new_line+'\n')
+
+def run_wc(args):
+    # if args.l==False and args.w==False and args.c==False:
+    #     args.l, args.w, args.c= True, True, True 
+    
+    nl=0
+    nw=0
+    nc=0
+
+    for line in args.infile:
+        nl+=1
+        nw+=len(line.strip(' ').split(' '))
+        nc+=len(line)
+
+    print('{} {} {}'.format(nl,nw,nc))
+
+def run_indic2roman(args):
+    for line in args.infile:
+        transliterated_line=unicode_transliterate.ItransTransliterator.to_itrans(
+            line,args.lang)
+        args.outfile.write(transliterated_line)      
+
+def run_roman2indic(args):
+    for line in args.infile:
+        transliterated_line=unicode_transliterate.ItransTransliterator.from_itrans(
+            line,args.lang)
+        args.outfile.write(transliterated_line)    
+
+def run_script_unify(args):
+
+    unifier=None
+
+    if args.mode=='aggressive':
+        unifier=script_unifier.AggressiveScriptUnifier(nasals_mode='to_anusvaara_relaxed', common_lang=args.common_lang)
+
+    elif args.mode=='basic':
+        unifier=script_unifier.BasicScriptUnifier(nasals_mode='do_nothing',
+                                common_lang=args.common_lang)
+
+    elif args.mode=='naive':
+        unifier=script_unifier.NaiveScriptUnifier(common_lang=args.common_lang)
+
+    assert(unifier is not None)
+
+    for line in args.infile:
+        transliterated_line=unifier.transform(line,args.lang)
+        args.outfile.write(transliterated_line)     
+
+def run_script_convert(args):
+    for line in args.infile:
+        transliterated_line=unicode_transliterate.UnicodeIndicTransliterator.transliterate(
+            line,args.srclang,args.tgtlang)
+        args.outfile.write(transliterated_line)    
+
+def add_common_monolingual_args(task_parser):
+    task_parser.add_argument('infile', 
+                type=argparse.FileType('r',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdin,
+                help='Input File path',
+            )
+    task_parser.add_argument('outfile', 
+                type=argparse.FileType('w',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdout,                
+                help='Output File path',
+            )
+    task_parser.add_argument('-l', '--lang', 
+                help='Language',
+            )
+
+def add_common_bilingual_args(task_parser):
+    task_parser.add_argument('infile', 
+                type=argparse.FileType('r',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdin,
+                help='Input File path',
+            )
+    task_parser.add_argument('outfile', 
+                type=argparse.FileType('w',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdout,                
+                help='Output File path',
+            )
+    task_parser.add_argument('-s', '--srclang', 
+                help='Source Language',
+            )
+
+    task_parser.add_argument('-t', '--tgtlang', 
+                help='Target Language',
+            )
+
+def add_tokenize_parser(subparsers):
+    task_parser=subparsers.add_parser('tokenize', 
+                    help='tokenizer help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_tokenize)
+
+def add_detokenize_parser(subparsers):
+    task_parser=subparsers.add_parser('detokenize', 
+                    help='de-tokenizer help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_detokenize)
+
+def add_sentence_split_parser(subparsers):
+    task_parser=subparsers.add_parser('sentence_split', help='sentence split help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_sentence_split)
+
+def add_normalize_parser(subparsers):
+    task_parser=subparsers.add_parser('normalize', help='normalizer help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_normalize)
+
+def add_morph_parser(subparsers):
+    task_parser=subparsers.add_parser('morph', help='morph help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_morph)
+
+def add_syllabify_parser(subparsers):
+    task_parser=subparsers.add_parser('syllabify', help='syllabify help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_syllabify)
+
+def add_wc_parser(subparsers):
+    task_parser=subparsers.add_parser('wc', help='wc help')
+
+    task_parser.add_argument('infile', 
+                type=argparse.FileType('r',encoding=DEFAULT_ENCODING),
+                nargs='?',
+                default=sys.stdin,
+                help='Input File path',
+            )
+    # task_parser.add_argument('-l', action='store_true')
+    # task_parser.add_argument('-w', action='store_true')
+    # task_parser.add_argument('-c', action='store_true')
+    # task_parser.set_defaults(l=False)
+    # task_parser.set_defaults(w=False)
+    # task_parser.set_defaults(c=False)
+
+    task_parser.set_defaults(func=run_wc)    
+
+def add_indic2roman_parser(subparsers):
+    task_parser=subparsers.add_parser('indic2roman', help='indic2roman help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_indic2roman)
+
+def add_roman2indic_parser(subparsers):
+    task_parser=subparsers.add_parser('roman2indic', help='roman2indic help')
+    add_common_monolingual_args(task_parser)
+    task_parser.set_defaults(func=run_indic2roman)
+
+def add_script_unify_parser(subparsers):
+    task_parser=subparsers.add_parser('script_unify', help='script_unify help')
+    add_common_monolingual_args(task_parser)
+    task_parser.add_argument('-m','--mode', 
+                default='basic',               
+                choices=['naive', 'basic', 'aggressive'] ,
+                help='Script unification mode',
+            )   
+    task_parser.add_argument('-c','--common_lang', 
+                default='hi',                
+                help='Common language in which all languages are represented',
+            )  
+
+    task_parser.set_defaults(func=run_script_unify)    
+
+def add_script_convert_parser(subparsers):
+    task_parser=subparsers.add_parser('script_convert', help='script convert help')
+    add_common_bilingual_args(task_parser)
+    task_parser.set_defaults(func=run_script_convert)
+
+def get_parser():
+    parser = argparse.ArgumentParser(prog='indicnlp')
+    subparsers = parser.add_subparsers(help='Invoke each operation with one of the subcommands', dest='subcommand')
+
+    add_tokenize_parser(subparsers)
+    add_detokenize_parser(subparsers)
+    add_sentence_split_parser(subparsers)
+    add_normalize_parser(subparsers)
+
+    add_morph_parser(subparsers)
+    add_syllabify_parser(subparsers)
+
+    add_wc_parser(subparsers)
+
+    add_indic2roman_parser(subparsers)
+    add_roman2indic_parser(subparsers)
+    add_script_unify_parser(subparsers)
+
+    add_script_convert_parser(subparsers)
+
+    return parser
+
+def main():
+    parser=get_parser()
+    args=parser.parse_args()
+    # print(args)
+    args.func(args)
+
+if __name__ == '__main__':
+    loader.load()
+    main()
+

indic_nlp_library/indicnlp/common.py

@@ -0,0 +1,58 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import os
+
+"""
+Path to the Indic NLP Resources directory
+"""
+INDIC_RESOURCES_PATH=''
+
+def init():
+    """
+    Initialize the module. The following actions are performed:
+
+    - Checks of INDIC_RESOURCES_PATH variable is set. If not, checks if it can beb initialized from 
+        INDIC_RESOURCES_PATH environment variable. If that fails, an exception is raised
+    """
+    global INDIC_RESOURCES_PATH 
+    try: 
+        if INDIC_RESOURCES_PATH=='':
+            INDIC_RESOURCES_PATH=os.environ['INDIC_RESOURCES_PATH']
+    except Exception as e: 
+        raise IndicNlpException('INDIC_RESOURCES_PATH not set')
+
+    if INDIC_RESOURCES_PATH=='': 
+        raise IndicNlpException('INDIC_RESOURCES_PATH not set')
+
+
+
+def get_resources_path(): 
+    """
+        Get the path to the Indic NLP Resources directory
+    """
+    return INDIC_RESOURCES_PATH
+
+def set_resources_path(resources_path): 
+    """
+        Set the path to the Indic NLP Resources directory
+    """
+    global INDIC_RESOURCES_PATH 
+    INDIC_RESOURCES_PATH=resources_path
+
+class IndicNlpException(Exception):
+    """
+        Exceptions thrown by Indic NLP Library components are instances of this class.  
+        'msg' attribute contains exception details.
+    """
+    def __init__(self, msg):
+        self.msg = msg 
+
+    def __str__(self):
+        return repr(self.msg)
+

indic_nlp_library/indicnlp/langinfo.py

@@ -0,0 +1,488 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+## language codes 
+LC_TA='ta'
+
+SCRIPT_RANGES={
+                 'pa':[0x0a00,0x0a7f] ,  
+                 'gu':[0x0a80,0x0aff] ,  
+                 'or':[0x0b00,0x0b7f] ,  
+                 'ta':[0x0b80,0x0bff] ,  
+                 'te':[0x0c00,0x0c7f] ,  
+                 'kn':[0x0c80,0x0cff] ,  
+                 'ml':[0x0d00,0x0d7f] ,  
+                 'si':[0x0d80,0x0dff] ,  
+                 'hi':[0x0900,0x097f] ,  
+                 'mr':[0x0900,0x097f] ,   
+                 'kK':[0x0900,0x097f] ,   
+                 'sa':[0x0900,0x097f] ,   
+                 'ne':[0x0900,0x097f] ,   
+                 'sd':[0x0900,0x097f] ,  
+                 'bn':[0x0980,0x09ff] ,  
+                 'as':[0x0980,0x09ff] ,  
+              }
+
+DRAVIDIAN_LANGUAGES=['ta', 'te', 'kn', 'ml',]
+IE_LANGUAGES=['hi', 'mr', 'kK', 'sa', 'ne', 'sd', 'bn', 'as', 'pa', 'gu', 'or', 'si', ]
+DANDA_DELIM_LANGUAGES=['as','bn','hi','ne','or','pa','sa','sd']
+
+URDU_RANGES=[
+                [0x0600,0x06ff], 
+                [0x0750,0x077f], 
+                [0xfb50,0xfdff], 
+                [0xfe70,0xfeff], 
+            ]
+
+COORDINATED_RANGE_START_INCLUSIVE=0
+COORDINATED_RANGE_END_INCLUSIVE=0x6f
+
+NUMERIC_OFFSET_START=0x66
+NUMERIC_OFFSET_END=0x6f
+
+HALANTA_OFFSET=0x4d
+AUM_OFFSET=0x50
+NUKTA_OFFSET=0x3c
+
+RUPEE_SIGN=0x20b9
+
+DANDA=0x0964
+DOUBLE_DANDA=0x0965
+
+#TODO: add missing fricatives and approximants
+VELAR_RANGE=[0x15,0x19]
+PALATAL_RANGE=[0x1a,0x1e]
+RETROFLEX_RANGE=[0x1f,0x23]
+DENTAL_RANGE=[0x24,0x29]
+LABIAL_RANGE=[0x2a,0x2e]
+
+# verify
+VOICED_LIST=[0x17,0x18,0x1c,0x1d,0x21,0x22,0x26,0x27,0x2c,0x2d]
+UNVOICED_LIST=[0x15,0x16,0x1a,0x1b,0x1f,0x20,0x24,0x25,0x2a,0x2b] #TODO: add sibilants/sonorants
+ASPIRATED_LIST=[0x16,0x18,0x1b,0x1d,0x20,0x22,0x25,0x27,0x2b,0x2d]
+UNASPIRATED_LIST=[0x15,0x17,0x1a,0x1c,0x1f,0x21,0x24,0x26,0x2a,0x2c]
+NASAL_LIST=[0x19,0x1e,0x23,0x28,0x29,0x2d]
+FRICATIVE_LIST=[0x36,0x37,0x38]
+APPROXIMANT_LIST=[0x2f,0x30,0x31,0x32,0x33,0x34,0x35]
+
+#TODO: ha has to be properly categorized 
+
+def is_danda_delim(lang):
+    """
+    Returns True if danda/double danda is a possible delimiter for the language
+    """
+    return lang in DANDA_DELIM_LANGUAGES
+
+def get_offset(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return ord(c)-SCRIPT_RANGES[lang][0]
+
+def offset_to_char(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return chr(c+SCRIPT_RANGES[lang][0])
+
+def in_coordinated_range(c_offset): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return  (c_offset>=COORDINATED_RANGE_START_INCLUSIVE and c_offset<=COORDINATED_RANGE_END_INCLUSIVE) 
+       
+def is_indiclang_char(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    o=get_offset(c,lang)
+    return (o>=0 and o<=0x7f) or ord(c)==DANDA or ord(c)==DOUBLE_DANDA
+
+# def is_vowel(c,lang): 
+#     """
+#     Is the character a vowel
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x04 and o<=0x14) 
+
+# def is_vowel_sign(c,lang): 
+#     """
+#     Is the character a vowel sign (maatraa)
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x3e and o<=0x4c) 
+
+# def is_halanta(c,lang): 
+#     """
+#     Is the character the halanta character
+#     """
+#     o=get_offset(c,lang)
+#     return (o==HALANTA_OFFSET) 
+
+# def is_nukta(c,lang): 
+#     """
+#     Is the character the halanta character
+#     """
+#     o=get_offset(c,lang)
+#     return (o==NUKTA_OFFSET) 
+
+# def is_aum(c,lang): 
+#     """
+#     Is the character a vowel sign (maatraa)
+#     """
+#     o=get_offset(c,lang)
+#     return (o==AUM_OFFSET)
+
+# def is_consonant(c,lang): 
+#     """
+#     Is the character a consonant
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x15 and o<=0x39) 
+
+# def is_velar(c,lang): 
+#     """
+#     Is the character a velar
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=VELAR_RANGE[0] and o<=VELAR_RANGE[1]) 
+
+# def is_palatal(c,lang): 
+#     """
+#     Is the character a palatal
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=PALATAL_RANGE[0] and o<=PALATAL_RANGE[1]) 
+
+# def is_retroflex(c,lang): 
+#     """
+#     Is the character a retroflex
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=RETROFLEX_RANGE[0] and o<=RETROFLEX_RANGE[1]) 
+
+# def is_dental(c,lang): 
+#     """
+#     Is the character a dental
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=DENTAL_RANGE[0] and o<=DENTAL_RANGE[1]) 
+
+# def is_labial(c,lang): 
+#     """
+#     Is the character a labial
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=LABIAL_RANGE[0] and o<=LABIAL_RANGE[1]) 
+
+# def is_voiced(c,lang): 
+#     """
+#     Is the character a voiced consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in VOICED_LIST
+
+# def is_unvoiced(c,lang): 
+#     """
+#     Is the character a unvoiced consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in UNVOICED_LIST
+
+# def is_aspirated(c,lang): 
+#     """
+#     Is the character a aspirated consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in ASPIRATED_LIST
+
+# def is_unaspirated(c,lang): 
+#     """
+#     Is the character a unaspirated consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in UNASPIRATED_LIST
+
+# def is_nasal(c,lang): 
+#     """
+#     Is the character a nasal consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in NASAL_LIST
+
+# def is_fricative(c,lang): 
+#     """
+#     Is the character a fricative consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in FRICATIVE_LIST
+
+# def is_approximant(c,lang): 
+#     """
+#     Is the character an approximant consonant
+#     """
+#     o=get_offset(c,lang)
+#     return o in APPROXIMANT_LIST
+
+# def is_number(c,lang): 
+#     """
+#     Is the character a number
+#     """
+#     o=get_offset(c,lang)
+#     return (o>=0x66 and o<=0x6f) 
+
+
+def is_vowel(c,lang): 
+    """
+    Is the character a vowel
+    """
+    o=get_offset(c,lang)
+    return (o>=0x04 and o<=0x14) 
+
+def is_vowel_sign(c,lang): 
+    """
+    Is the character a vowel sign (maatraa)
+    """
+    o=get_offset(c,lang)
+    return (o>=0x3e and o<=0x4c) 
+
+def is_halanta(c,lang): 
+    """
+    Is the character the halanta character
+    """
+    o=get_offset(c,lang)
+    return (o==HALANTA_OFFSET) 
+
+def is_nukta(c,lang): 
+    """
+    Is the character the halanta character
+    """
+    o=get_offset(c,lang)
+    return (o==NUKTA_OFFSET) 
+
+def is_aum(c,lang): 
+    """
+    Is the character a vowel sign (maatraa)
+    """
+    o=get_offset(c,lang)
+    return (o==AUM_OFFSET)
+
+def is_consonant(c,lang): 
+    """
+    Is the character a consonant
+    """
+    o=get_offset(c,lang)
+    return (o>=0x15 and o<=0x39) 
+
+def is_velar(c,lang): 
+    """
+    Is the character a velar
+    """
+    o=get_offset(c,lang)
+    return (o>=VELAR_RANGE[0] and o<=VELAR_RANGE[1]) 
+
+def is_palatal(c,lang): 
+    """
+    Is the character a palatal
+    """
+    o=get_offset(c,lang)
+    return (o>=PALATAL_RANGE[0] and o<=PALATAL_RANGE[1]) 
+
+def is_retroflex(c,lang): 
+    """
+    Is the character a retroflex
+    """
+    o=get_offset(c,lang)
+    return (o>=RETROFLEX_RANGE[0] and o<=RETROFLEX_RANGE[1]) 
+
+def is_dental(c,lang): 
+    """
+    Is the character a dental
+    """
+    o=get_offset(c,lang)
+    return (o>=DENTAL_RANGE[0] and o<=DENTAL_RANGE[1]) 
+
+def is_labial(c,lang): 
+    """
+    Is the character a labial
+    """
+    o=get_offset(c,lang)
+    return (o>=LABIAL_RANGE[0] and o<=LABIAL_RANGE[1]) 
+
+def is_voiced(c,lang): 
+    """
+    Is the character a voiced consonant
+    """
+    o=get_offset(c,lang)
+    return o in VOICED_LIST
+
+def is_unvoiced(c,lang): 
+    """
+    Is the character a unvoiced consonant
+    """
+    o=get_offset(c,lang)
+    return o in UNVOICED_LIST
+
+def is_aspirated(c,lang): 
+    """
+    Is the character a aspirated consonant
+    """
+    o=get_offset(c,lang)
+    return o in ASPIRATED_LIST
+
+def is_unaspirated(c,lang): 
+    """
+    Is the character a unaspirated consonant
+    """
+    o=get_offset(c,lang)
+    return o in UNASPIRATED_LIST
+
+def is_nasal(c,lang): 
+    """
+    Is the character a nasal consonant
+    """
+    o=get_offset(c,lang)
+    return o in NASAL_LIST
+
+def is_fricative(c,lang): 
+    """
+    Is the character a fricative consonant
+    """
+    o=get_offset(c,lang)
+    return o in FRICATIVE_LIST
+
+def is_approximant(c,lang): 
+    """
+    Is the character an approximant consonant
+    """
+    o=get_offset(c,lang)
+    return o in APPROXIMANT_LIST
+
+def is_number(c,lang): 
+    """
+    Is the character a number
+    """
+    o=get_offset(c,lang)
+    return (o>=0x66 and o<=0x6f) 
+
+
+##################################################
+
+def is_vowel_offset(c_offset):
+    """
+    Is the offset a vowel
+    """
+    return (c_offset>=0x04 and c_offset<=0x14) 
+
+def is_vowel_sign_offset(c_offset):
+    """
+    Is the offset a vowel sign (maatraa)
+    """
+    return (c_offset>=0x3e and c_offset<=0x4c) 
+
+def is_halanta_offset(c_offset):
+    """
+    Is the offset the halanta offset
+    """
+    return (c_offset==HALANTA_OFFSET) 
+
+def is_nukta_offset(c_offset):
+    """
+    Is the offset the halanta offset
+    """
+    return (c_offset==NUKTA_OFFSET) 
+
+def is_aum_offset(c_offset):
+    """
+    Is the offset a vowel sign (maatraa)
+    """
+    return (c_offset==AUM_OFFSET)
+
+def is_consonant_offset(c_offset):
+    """
+    Is the offset a consonant
+    """
+    return (c_offset>=0x15 and c_offset<=0x39) 
+
+def is_velar_offset(c_offset):
+    """
+    Is the offset a velar
+    """
+    return (c_offset>=VELAR_RANGE[0] and c_offset<=VELAR_RANGE[1]) 
+
+def is_palatal_offset(c_offset):
+    """
+    Is the offset a palatal
+    """
+    return (c_offset>=PALATAL_RANGE[0] and c_offset<=PALATAL_RANGE[1]) 
+
+def is_retroflex_offset(c_offset):
+    """
+    Is the offset a retroflex
+    """
+    return (c_offset>=RETROFLEX_RANGE[0] and c_offset<=RETROFLEX_RANGE[1]) 
+
+def is_dental_offset(c_offset):
+    """
+    Is the offset a dental
+    """
+    return (c_offset>=DENTAL_RANGE[0] and c_offset<=DENTAL_RANGE[1]) 
+
+def is_labial_offset(c_offset):
+    """
+    Is the offset a labial
+    """
+    return (c_offset>=LABIAL_RANGE[0] and c_offset<=LABIAL_RANGE[1]) 
+
+def is_voiced_offset(c_offset):
+    """
+    Is the offset a voiced consonant
+    """
+    return c_offset in VOICED_LIST
+
+def is_unvoiced_offset(c_offset):
+    """
+    Is the offset a unvoiced consonant
+    """
+    return c_offset in UNVOICED_LIST
+
+def is_aspirated_offset(c_offset):
+    """
+    Is the offset a aspirated consonant
+    """
+    return c_offset in ASPIRATED_LIST
+
+def is_unaspirated_offset(c_offset):
+    """
+    Is the offset a unaspirated consonant
+    """
+    return c_offset in UNASPIRATED_LIST
+
+def is_nasal_offset(c_offset):
+    """
+    Is the offset a nasal consonant
+    """
+    return c_offset in NASAL_LIST
+
+def is_fricative_offset(c_offset):
+    """
+    Is the offset a fricative consonant
+    """
+    return c_offset in FRICATIVE_LIST
+
+def is_approximant_offset(c_offset):
+    """
+    Is the offset an approximant consonant
+    """
+    return c_offset in APPROXIMANT_LIST
+
+def is_number_offset(c_offset):
+    """
+    Is the offset a number
+    """
+    return (c_offset>=0x66 and c_offset<=0x6f) 

indic_nlp_library/indicnlp/loader.py

@@ -0,0 +1,35 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+from indicnlp import common
+from indicnlp.script import indic_scripts
+from indicnlp.script import english_script
+from indicnlp.transliterate import unicode_transliterate
+
+def load():
+    """
+        Initializes the Indic NLP library. Clients should call this method before using the library. 
+
+        Any module requiring initialization should have a init() method, to which a call must be made from this method 
+    """
+
+    ### Order of intialization may matter 
+
+    # Common has to be loaded first to get access to resources 
+    common.init()
+
+    ## Initialization of Indic scripts module 
+    indic_scripts.init()
+
+    ## Initialization of English scripts module 
+    english_script.init()
+
+    ## Initialization of unicode_transliterate module 
+    unicode_transliterate.init()
+
+

indic_nlp_library/indicnlp/morph/unsupervised_morph.py

@@ -0,0 +1,142 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import codecs, sys, itertools,re,os
+import morfessor 
+
+from functools import lru_cache
+
+from indicnlp import langinfo
+from indicnlp import common
+from indicnlp.tokenize import indic_tokenize
+
+# Unsupervised Morphological Analyser for Indian languages. 
+#
+# @author Anoop Kunchukuttan 
+#
+
+class MorphAnalyzerI(object):
+    """
+     Interface for Morph Analyzer
+    """
+
+    def morph_analyze(word):
+        pass 
+
+    def morph_analyze_document(tokens):
+        pass 
+
+class UnsupervisedMorphAnalyzer(MorphAnalyzerI): 
+    """
+    Unsupervised Morphological analyser built using Morfessor 2.0
+    """
+
+    def __init__(self,lang,add_marker=False): 
+        self.lang=lang
+        self.add_marker=add_marker
+
+        io = morfessor.MorfessorIO()
+        self._morfessor_model=io.read_any_model(os.path.join(common.INDIC_RESOURCES_PATH,'morph','morfessor','{}.model'.format(lang)))        
+
+        self._script_range_pat=r'^[{}-{}]+$'.format(chr(langinfo.SCRIPT_RANGES[lang][0]),chr(langinfo.SCRIPT_RANGES[lang][1]))
+        self._script_check_re=re.compile(self._script_range_pat)
+
+    def _contains_number(self,text):
+        if self.lang in langinfo.SCRIPT_RANGES: 
+            for c in text: 
+                offset=ord(c)-langinfo.SCRIPT_RANGES[self.lang][0]
+                if offset >=langinfo.NUMERIC_OFFSET_START and offset <= langinfo.NUMERIC_OFFSET_END:
+                    return True  
+        return False     
+
+    def _morphanalysis_needed(self,word):
+        return self._script_check_re.match(word) and not self._contains_number(word)
+
+    @lru_cache(maxsize=16384)
+    def morph_analyze(self,word):
+        """
+        Morphanalyzes a single word and returns a list of component morphemes
+
+        @param word: string input word 
+        """
+        m_list=[]
+        if self._morphanalysis_needed(word): 
+            val=self._morfessor_model.viterbi_segment(word)
+            m_list=val[0]
+            if self.add_marker:
+                m_list= [ '{}_S_'.format(m) if i>0 else '{}_R_'.format(m)  for i,m in enumerate(m_list)]
+        else:
+            if self.add_marker:
+                word='{}_E_'.format(word)
+            m_list=[word]
+        return m_list 
+
+        ### Older implementation
+        #val=self._morfessor_model.viterbi_segment(word)
+        #m_list=val[0]
+        #if self.add_marker:
+        #    m_list= [ u'{}_S_'.format(m) if i>0 else u'{}_R_'.format(m)  for i,m in enumerate(m_list)]
+        #return m_list
+    
+
+    def morph_analyze_document(self,tokens):
+        """
+        Morphanalyzes a document, represented as a list of tokens
+        Each word  is morphanalyzed and result is a list of morphemes constituting the document 
+
+        @param tokens: string sequence of words 
+
+        @return list of segments in the document after morph analysis 
+        """
+
+        out_tokens=[]
+        for token in tokens: 
+            morphs=self.morph_analyze(token)
+            out_tokens.extend(morphs)
+        return out_tokens    
+
+        #### Older implementation
+        #out_tokens=[]
+        #for token in tokens: 
+        #    if self._morphanalysis_needed(token): 
+        #        morphs=self.morph_analyze(token)
+        #        out_tokens.extend(morphs)
+        #    else:
+        #        if self.add_marker:
+        #            token=u'{}_E_'.format(token)
+        #        out_tokens.append(token)
+        #return out_tokens    
+
+
+if __name__ == '__main__': 
+
+    if len(sys.argv)<4:
+        print("Usage: python unsupervised_morph.py <infile> <outfile> <language> <indic_resources_path> [<add_marker>]")
+        sys.exit(1)
+
+    language=sys.argv[3]
+    common.INDIC_RESOURCES_PATH=sys.argv[4]
+
+    add_marker=False
+
+    if len(sys.argv)==6:
+        add_marker= True if sys.argv[5] == 'True' else False 
+
+    print('Loading morph analyser for ' + language) 
+    analyzer=UnsupervisedMorphAnalyzer(language,add_marker)
+    print('Loaded morph analyser for ' + language) 
+
+    with codecs.open(sys.argv[1],'r','utf-8') as ifile:
+        with codecs.open(sys.argv[2],'w','utf-8') as ofile:
+            for line in ifile.readlines():
+                line=line.strip()
+                tokens=indic_tokenize.trivial_tokenize(line)
+                morph_tokens=analyzer.morph_analyze_document(tokens)
+                ofile.write(' '.join(morph_tokens))
+                ofile.write('\n')
+

indic_nlp_library/indicnlp/normalize/indic_normalize.py

@@ -0,0 +1,984 @@
+# -*- coding: utf-8 -*-
+
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+#Program for normalization of text written in Unicode. This is mainly geared towards Indic scripts 
+#
+# @author Anoop Kunchukuttan 
+#
+
+import sys, codecs, string, itertools, re
+from indicnlp import langinfo
+
+
+class NormalizerI(object):
+    """
+    The normalizer classes do the following: 
+
+    * Some characters have multiple Unicode codepoints. The normalizer chooses a single standard representation
+    * Some control characters are deleted
+    * While typing using the Latin keyboard, certain typical mistakes occur which are corrected by the module
+
+    Base class for normalizer. Performs some common normalization, which includes: 
+
+    * Byte order mark, word joiner, etc. removal 
+    * ZERO_WIDTH_NON_JOINER and ZERO_WIDTH_JOINER removal 
+    * ZERO_WIDTH_SPACE and NO_BREAK_SPACE replaced by spaces 
+
+    Script specific normalizers should derive from this class and override the normalize() method. 
+    They can call the super class 'normalize() method to avail of the common normalization 
+
+    """
+
+    BYTE_ORDER_MARK='\uFEFF'
+    BYTE_ORDER_MARK_2='\uFFFE'
+    WORD_JOINER='\u2060'
+    SOFT_HYPHEN='\u00AD'
+
+    ZERO_WIDTH_SPACE='\u200B'
+    NO_BREAK_SPACE='\u00A0'
+
+    ZERO_WIDTH_NON_JOINER='\u200C'
+    ZERO_WIDTH_JOINER='\u200D'
+
+    def _normalize_punctuations(self, text):
+        """
+        Normalize punctuations. 
+        Applied many of the punctuation normalizations that are part of MosesNormalizer 
+        from sacremoses
+        """
+        text=text.replace(NormalizerI.BYTE_ORDER_MARK,'')
+        text=text.replace('„', r'"')
+        text=text.replace('“', r'"')
+        text=text.replace('”', r'"')
+        text=text.replace('–', r'-')
+        text=text.replace('—', r' - ')
+        text=text.replace('´', r"'")
+        text=text.replace('‘', r"'")
+        text=text.replace('‚', r"'")
+        text=text.replace('’', r"'")
+        text=text.replace("''", r'"')
+        text=text.replace('´´', r'"')
+        text=text.replace('…', r'...')
+
+        return text
+
+    def normalize(self,text):
+        pass 
+
+
+class BaseNormalizer(NormalizerI):
+
+    def __init__(self,lang,
+            remove_nuktas=False,
+            nasals_mode='do_nothing',
+            do_normalize_chandras=False,
+            do_normalize_vowel_ending=False):
+
+        self.lang=lang
+        self.remove_nuktas=remove_nuktas
+        self.nasals_mode=nasals_mode
+        self.do_normalize_chandras=do_normalize_chandras
+        self.do_normalize_vowel_ending=do_normalize_vowel_ending
+
+        self._init_normalize_chandras()
+        self._init_normalize_nasals()
+        self._init_normalize_vowel_ending()
+        #self._init_visarga_correction()
+        
+    def _init_normalize_vowel_ending(self):
+
+        if self.lang in langinfo.IE_LANGUAGES:
+            self.fn_vowel_ending=self._normalize_word_vowel_ending_ie
+        elif self.lang in langinfo.DRAVIDIAN_LANGUAGES:
+            self.fn_vowel_ending=self._normalize_word_vowel_ending_dravidian
+        else:
+            self.fn_vowel_ending=lambda x: x
+
+    def _init_normalize_chandras(self):
+
+        substitution_offsets =\
+            [
+                [0x0d , 0x0f], # chandra e, independent
+                [0x11 , 0x13], # chandra o, independent
+                [0x45 , 0x47], # chandra e , 0xde],pendent
+                [0x49 , 0x4b], # chandra o , 0xde],pendent
+                # [0x72 , 0x0f], # mr: chandra e, independent
+
+                [0x00 , 0x02], # chandrabindu
+                [0x01 , 0x02], # chandrabindu
+            ]
+
+        self.chandra_substitutions =  [ 
+                (langinfo.offset_to_char(x[0],self.lang), langinfo.offset_to_char(x[1],self.lang)) 
+                    for x in substitution_offsets ]
+
+    def _normalize_chandras(self,text):
+        for match, repl in self.chandra_substitutions:
+            text=text.replace(match,repl)
+        return text
+
+    def _init_to_anusvaara_strict(self):
+        """
+        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
+        """
+    
+        pat_signatures=\
+            [
+                 [0x19,0x15,0x18],
+                 [0x1e,0x1a,0x1d],            
+                 [0x23,0x1f,0x22],                        
+                 [0x28,0x24,0x27],        
+                 [0x29,0x24,0x27],                    
+                 [0x2e,0x2a,0x2d],                    
+            ]    
+        
+        halant_offset=0x4d
+        anusvaara_offset=0x02
+        
+        pats=[]
+        
+        for pat_signature in pat_signatures:
+            pat=re.compile(r'{nasal}{halant}([{start_r}-{end_r}])'.format(
+                nasal=langinfo.offset_to_char(pat_signature[0],self.lang),
+                halant=langinfo.offset_to_char(halant_offset,self.lang),
+                start_r=langinfo.offset_to_char(pat_signature[1],self.lang),
+                end_r=langinfo.offset_to_char(pat_signature[2],self.lang),
+            ))
+            pats.append(pat)
+        
+        repl_string='{anusvaara}\\1'.format(anusvaara=langinfo.offset_to_char(anusvaara_offset,self.lang))
+
+        self.pats_repls=(pats,repl_string)
+    
+    def _to_anusvaara_strict(self,text):
+        
+        pats, repl_string = self.pats_repls
+        for pat in pats:
+            text=pat.sub(repl_string,text)
+            
+        return text
+
+    def _init_to_anusvaara_relaxed(self):
+        """
+        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
+        """
+            
+        nasals_list=[0x19,0x1e,0x23,0x28,0x29,0x2e]    
+        nasals_list_str=','.join([langinfo.offset_to_char(x,self.lang) for x in nasals_list])
+        
+        halant_offset=0x4d    
+        anusvaara_offset=0x02    
+        
+        pat=re.compile(r'[{nasals_list_str}]{halant}'.format(
+                nasals_list_str=nasals_list_str,
+                halant=langinfo.offset_to_char(halant_offset,self.lang),
+            ))
+        
+        repl_string='{anusvaara}'.format(anusvaara=langinfo.offset_to_char(anusvaara_offset,self.lang))
+
+        self.pats_repls = (pat,repl_string)
+    
+    def _to_anusvaara_relaxed(self,text):
+        pat, repl_string = self.pats_repls
+        return pat.sub(repl_string,text)
+    
+
+    def _init_to_nasal_consonants(self):
+        """
+        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
+        """
+    
+        pat_signatures=\
+            [
+                 [0x19,0x15,0x18],
+                 [0x1e,0x1a,0x1d],            
+                 [0x23,0x1f,0x22],                        
+                 [0x28,0x24,0x27],        
+                 [0x29,0x24,0x27],                    
+                 [0x2e,0x2a,0x2d],                    
+            ]    
+        
+        halant_offset=0x4d
+        anusvaara_offset=0x02 
+        
+        pats=[]
+        repl_strings=[]
+        
+        for pat_signature in pat_signatures:
+            pat=re.compile(r'{anusvaara}([{start_r}-{end_r}])'.format(
+                anusvaara=langinfo.offset_to_char(anusvaara_offset,self.lang),
+                start_r=langinfo.offset_to_char(pat_signature[1],self.lang),
+                end_r=langinfo.offset_to_char(pat_signature[2],self.lang),
+            ))
+            pats.append(pat)
+            repl_string='{nasal}{halant}\\1'.format(
+                nasal=langinfo.offset_to_char(pat_signature[0],self.lang),
+                halant=langinfo.offset_to_char(halant_offset,self.lang),
+                )
+            repl_strings.append(repl_string)
+    
+        self.pats_repls=list(zip(pats,repl_strings))
+
+    def _to_nasal_consonants(self,text):
+    
+        for pat, repl in self.pats_repls:
+            text=pat.sub(repl,text)
+            
+        return text
+
+    def _init_normalize_nasals(self):
+
+        if self.nasals_mode == 'to_anusvaara_strict':
+            self._init_to_anusvaara_strict()
+        elif self.nasals_mode == 'to_anusvaara_relaxed':
+            self._init_to_anusvaara_relaxed()
+        elif self.nasals_mode == 'to_nasal_consonants':
+            self._init_to_nasal_consonants()
+
+    def _normalize_nasals(self,text): 
+        if self.nasals_mode == 'to_anusvaara_strict':
+            return self._to_anusvaara_strict(text)
+        elif self.nasals_mode == 'to_anusvaara_relaxed':
+            return self._to_anusvaara_relaxed(text)
+        elif self.nasals_mode == 'to_nasal_consonants':
+            return self._to_nasal_consonants(text)
+        else:
+            return text
+
+    
+    def _normalize_word_vowel_ending_dravidian(self,word):
+        """
+        for Dravidian
+        - consonant ending: add 'a' ki maatra
+        - halant ending: no change
+        - 'a' ki maatra: no change
+        """
+        if len(word)>0 and langinfo.is_consonant(word[-1],self.lang):
+            return word+langinfo.offset_to_char(0x3e,self.lang)
+        else:
+            return word
+
+    def _normalize_word_vowel_ending_ie(self,word):
+        """
+        for IE
+        - consonant ending: add halant
+        - halant ending: no change
+        - 'a' ki maatra: no change
+        """
+        if len(word)>0 and langinfo.is_consonant(word[-1],self.lang):
+            return word+langinfo.offset_to_char(langinfo.HALANTA_OFFSET,self.lang)
+        else:
+            return word 
+
+    def _normalize_vowel_ending(self,text):
+        return ' '.join([ self.fn_vowel_ending(w) for w in text.split(' ') ])
+
+    def normalize(self,text):
+        """
+        Method to be implemented for normalization for each script 
+        """
+        text=text.replace(NormalizerI.BYTE_ORDER_MARK,'')
+        text=text.replace(NormalizerI.BYTE_ORDER_MARK_2,'')
+        text=text.replace(NormalizerI.WORD_JOINER,'')
+        text=text.replace(NormalizerI.SOFT_HYPHEN,'')
+
+        text=text.replace(NormalizerI.ZERO_WIDTH_SPACE,' ') # ??
+        text=text.replace(NormalizerI.NO_BREAK_SPACE,' ')
+
+        text=text.replace(NormalizerI.ZERO_WIDTH_NON_JOINER, '')
+        text=text.replace(NormalizerI.ZERO_WIDTH_JOINER,'')
+        
+        text=self._normalize_punctuations(text)
+
+        if self.do_normalize_chandras:
+            text=self._normalize_chandras(text)
+        text=self._normalize_nasals(text)
+        if self.do_normalize_vowel_ending:
+            text=self._normalize_vowel_ending(text)
+        
+        return text
+
+
+    def get_char_stats(self,text):    
+        print(len(re.findall(NormalizerI.BYTE_ORDER_MARK,text)))
+        print(len(re.findall(NormalizerI.BYTE_ORDER_MARK_2,text)))
+        print(len(re.findall(NormalizerI.WORD_JOINER,text)))
+        print(len(re.findall(NormalizerI.SOFT_HYPHEN,text)))
+
+        print(len(re.findall(NormalizerI.ZERO_WIDTH_SPACE,text) ))
+        print(len(re.findall(NormalizerI.NO_BREAK_SPACE,text)))
+
+        print(len(re.findall(NormalizerI.ZERO_WIDTH_NON_JOINER,text)))
+        print(len(re.findall(NormalizerI.ZERO_WIDTH_JOINER,text)))
+
+        #for mobj in re.finditer(NormalizerI.ZERO_WIDTH_NON_JOINER,text):
+        #    print text[mobj.start()-10:mobj.end()+10].replace('\n', ' ').replace(NormalizerI.ZERO_WIDTH_NON_JOINER,'').encode('utf-8')
+        #print hex(ord(text[mobj.end():mobj.end()+1]))
+
+    def correct_visarga(self,text,visarga_char,char_range):
+        text=re.sub(r'([\u0900-\u097f]):','\\1\u0903',text)
+        
+
+
+class DevanagariNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Devanagari script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    
+    """
+
+    NUKTA='\u093C' 
+
+    def __init__(self,lang='hi',remove_nuktas=False,nasals_mode='do_nothing',
+            do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(DevanagariNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(DevanagariNormalizer,self).normalize(text)
+
+        # chandra a replacement for Marathi
+        text=text.replace('\u0972','\u090f')
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u0929','\u0928'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0931','\u0930'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0934','\u0933'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0958','\u0915'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u0959','\u0916'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095A','\u0917'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095B','\u091C'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095C','\u0921'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095D','\u0922'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095E','\u092B'+DevanagariNormalizer.NUKTA)
+        text=text.replace('\u095F','\u092F'+DevanagariNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(DevanagariNormalizer.NUKTA,'')
+
+        # replace pipe character for poorna virama 
+        text=text.replace('\u007c','\u0964')
+
+        # correct visarga 
+        text=re.sub(r'([\u0900-\u097f]):','\\1\u0903',text)
+
+        return text
+
+    def get_char_stats(self,text):
+        super(DevanagariNormalizer,self).get_char_stats(text)
+
+        print((len(re.findall('\u0929',text))))
+        print((len(re.findall('\u0931',text))))
+        print((len(re.findall('\u0934',text))))
+        print((len(re.findall('\u0958',text))))
+        print((len(re.findall('\u0959',text))))
+        print((len(re.findall('\u095A',text))))
+        print((len(re.findall('\u095B',text))))
+        print((len(re.findall('\u095C',text))))
+        print((len(re.findall('\u095D',text))))
+        print((len(re.findall('\u095E',text))))
+        print((len(re.findall('\u095F',text))))
+
+        #print(len(re.findall(u'\u0928'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0930'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0933'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0915'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0916'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0917'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u091C'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0921'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u0922'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u092B'+DevanagariNormalizer.NUKTA,text)))
+        #print(len(re.findall(u'\u092F'+DevanagariNormalizer.NUKTA,text)))
+
+class GurmukhiNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Gurmukhi script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    NUKTA='\u0A3C' 
+
+    VOWEL_NORM_MAPS={
+        ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
+        ## Table 12-16
+        '\u0a05\u0a3e': '\u0a06',
+        '\u0a72\u0a3f': '\u0a07',
+        '\u0a72\u0a40': '\u0a08',
+        '\u0a73\u0a41': '\u0a09',
+        '\u0a73\u0a42': '\u0a0a',
+        '\u0a72\u0a47': '\u0a0f',
+        '\u0a05\u0a48': '\u0a10',
+        '\u0a73\u0a4b': '\u0a13',
+        '\u0a05\u0a4c': '\u0a14',            
+    }
+
+    def __init__(self,lang='pa',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                do_normalize_vowel_ending=False,
+                do_canonicalize_addak=False, 
+                do_canonicalize_tippi=False, 
+                do_replace_vowel_bases=False):
+        super(GurmukhiNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_canonicalize_addak=do_canonicalize_addak
+        self.do_canonicalize_tippi=do_canonicalize_tippi
+        self.do_replace_vowel_bases=do_replace_vowel_bases
+
+
+    def _normalize_vowels(self,text):
+        """
+
+        """
+
+        ## standard vowel replacements as per suggestions in 
+        ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
+        ## Table 12-16
+
+        for k,v in GurmukhiNormalizer.VOWEL_NORM_MAPS.items():
+            text=text.replace(k,v)
+        
+        ## the above mappings should account for majority of the variantions, 
+        ## Rest are handled via this generic rule which looks at the diacritic 
+        ## following the 2 special characters 
+        ## TBD: don't see evidence for this in Wikipedia corpus
+
+        ## If these special characters occur without any diacritic, replace them with closet
+        ## equivalent vowels
+        if self.do_replace_vowel_bases:
+            text=text.replace('\u0a72','\u0a07')
+            text=text.replace('\u0a73','\u0a09')
+
+        return text
+
+
+    def normalize(self,text): 
+
+        # Addak
+        if self.do_canonicalize_addak:
+            ## replace addak+consonant with consonat+halant+consonant
+            text=re.sub(r'\u0a71(.)','\\1\u0a4d\\1',text)
+            
+        # Tippi 
+        if self.do_canonicalize_tippi:
+            text=text.replace('\u0a70','\u0a02') 
+
+        # Vowels: Gurumuki has multiple ways of representing independent vowels due
+        # to the characters 'iri' and 'ura'. 
+        text=self._normalize_vowels(text)
+
+        # common normalization for Indic scripts 
+        text=super(GurmukhiNormalizer,self).normalize(text)
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u0a33','\u0a32'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a36','\u0a38'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a59','\u0a16'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a5a','\u0a17'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a5b','\u0a1c'+GurmukhiNormalizer.NUKTA)
+        text=text.replace('\u0a5e','\u0a2b'+GurmukhiNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(GurmukhiNormalizer.NUKTA,'')
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0a64','\u0964')
+        text=text.replace('\u0a65','\u0965')
+
+        ## replace pipe character for poorna virama 
+        text=text.replace('\u007c','\u0964')
+
+        # correct visarge 
+        text=re.sub(r'([\u0a00-\u0a7f]):','\\1\u0a03',text)
+
+        return text
+
+
+class GujaratiNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Gujarati script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    NUKTA='\u0ABC' 
+
+    def __init__(self,lang='gu',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                    do_normalize_vowel_ending=False):
+        super(GujaratiNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(GujaratiNormalizer,self).normalize(text)
+
+        # decomposing Nukta based composite characters
+        if self.remove_nuktas:
+            text=text.replace(GujaratiNormalizer.NUKTA,'')
+
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0ae4','\u0964')
+        text=text.replace('\u0ae5','\u0965')
+
+        # correct visarge 
+        text=re.sub(r'([\u0a80-\u0aff]):','\\1\u0a83',text)
+
+        return text
+
+
+class OriyaNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Oriya script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * Canonicalize two part dependent vowels
+    * Replace 'va' with 'ba'
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    NUKTA='\u0B3C' 
+
+    VOWEL_NORM_MAPS={
+        ## See Table 12-22 in http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
+        '\u0b05\u0b3e': '\u0b06',
+        '\u0b0f\u0b57': '\u0b10',
+        '\u0b13\u0b57': '\u0b14',
+    }
+
+
+    def __init__(self,lang='or',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                do_normalize_vowel_ending=False,
+                do_remap_wa=False):
+        super(OriyaNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_remap_wa=do_remap_wa
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(OriyaNormalizer,self).normalize(text)
+
+        ## standard vowel replacements as per suggestions in Unicode documents
+        for k,v in OriyaNormalizer.VOWEL_NORM_MAPS.items():
+            text=text.replace(k,v)
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u0b5c','\u0b21'+OriyaNormalizer.NUKTA)
+        text=text.replace('\u0b5d','\u0b22'+OriyaNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(OriyaNormalizer.NUKTA,'')
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0b64','\u0964')
+        text=text.replace('\u0b65','\u0965')
+
+        # replace pipe character for poorna virama 
+        text=text.replace('\u0b7c','\u0964')
+
+        # replace wa with ba 
+        if self.do_remap_wa:
+            text=text.replace('\u0b71','\u0b2c')
+
+        # replace va with ba 
+        # NOTE: documentation (chapter on Indic scripts) and codepoint chart seem contradictory 
+        # (this applied to wa to ba rule also above)
+        text=text.replace('\u0b35','\u0b2c')
+
+        # AI dependent vowel sign 
+        text=text.replace('\u0b47\u0b56','\u0b58')
+
+        # two part dependent vowels
+        text=text.replace('\u0b47\u0b3e','\u0b4b')
+        text=text.replace('\u0b47\u0b57','\u0b4c')
+
+
+        # additional consonant - not clear how to handle this
+        # ignore
+
+        # correct visarge 
+        text=re.sub(r'([\u0b00-\u0b7f]):','\\1\u0b03',text)
+
+        return text
+
+
+class BengaliNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Bengali script. In addition to basic normalization by the super class, 
+
+    * Replaces the composite characters containing nuktas by their decomposed form 
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * Canonicalize two part dependent vowels
+    * replace pipe character '|' by poorna virama character
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+
+    """
+
+    NUKTA='\u09BC' 
+
+    def __init__(self,lang='bn',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                    do_normalize_vowel_ending=False,
+                    do_remap_assamese_chars=False):
+        super(BengaliNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_remap_assamese_chars=do_remap_assamese_chars
+
+    def normalize(self,text):
+
+        # common normalization for Indic scripts 
+        text=super(BengaliNormalizer,self).normalize(text)
+
+        # decomposing Nukta based composite characters
+        text=text.replace('\u09dc','\u09a1'+BengaliNormalizer.NUKTA)
+        text=text.replace('\u09dd','\u09a2'+BengaliNormalizer.NUKTA)
+        text=text.replace('\u09df','\u09af'+BengaliNormalizer.NUKTA)
+
+        if self.remove_nuktas:
+            text=text.replace(BengaliNormalizer.NUKTA,'')
+
+        if self.do_remap_assamese_chars and self.lang=='as':
+            text=text.replace('\u09f0','\u09b0')  #  'ra' character
+            text=text.replace('\u09f1','\u09ac')  #  'va' character 
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u09e4','\u0964')
+        text=text.replace('\u09e5','\u0965')
+
+        # replace pipe character for poorna virama 
+        text=text.replace('\u007c','\u0964')
+        # replace bengali currency numerator four for poorna virama  (it looks similar and is used as a substitute)
+        text=text.replace('\u09f7','\u0964')
+
+        # two part dependent vowels
+        text=text.replace('\u09c7\u09be','\u09cb')
+        text=text.replace('\u09c7\u09d7','\u09cc')
+
+        # correct visarge 
+        text=re.sub(r'([\u0980-\u09ff]):','\\1\u0983',text)
+
+        return text
+
+
+class TamilNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Tamil script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    def __init__(self,lang='ta',remove_nuktas=False,nasals_mode='do_nothing',
+            do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(TamilNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(TamilNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0be4','\u0964')
+        text=text.replace('\u0be5','\u0965')
+
+        # two part dependent vowels
+        text=text.replace('\u0b92\u0bd7','\u0b94')
+        text=text.replace('\u0bc6\u0bbe','\u0bca')
+        text=text.replace('\u0bc7\u0bbe','\u0bcb')
+        text=text.replace('\u0bc6\u0bd7','\u0bcc')
+
+        # correct visarge 
+        text=re.sub(r'([\u0b80-\u0bff]):','\\1\u0b83',text)
+
+        return text
+
+
+class TeluguNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Teluguscript. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    def __init__(self,lang='te',remove_nuktas=False,nasals_mode='do_nothing',
+                do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(TeluguNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(TeluguNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0c64','\u0964')
+        text=text.replace('\u0c65','\u0965')
+
+        # dependent vowels
+        text=text.replace('\u0c46\u0c56','\u0c48')
+
+        # correct visarge 
+        text=re.sub(r'([\u0c00-\u0c7f]):','\\1\u0c03',text)
+
+        return text
+
+    def get_char_stats(self,text):
+        pass 
+
+class KannadaNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Kannada script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    def __init__(self,lang='kn',remove_nuktas=False,nasals_mode='do_nothing',
+            do_normalize_chandras=False,do_normalize_vowel_ending=False):
+        super(KannadaNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+
+
+    def normalize(self,text): 
+
+        # common normalization for Indic scripts 
+        text=super(KannadaNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0ce4','\u0964')
+        text=text.replace('\u0ce5','\u0965')
+
+        # dependent vowels
+        text=text.replace('\u0cbf\u0cd5','\u0cc0')
+        text=text.replace('\u0cc6\u0cd5','\u0cc7')
+        text=text.replace('\u0cc6\u0cd6','\u0cc8')
+        text=text.replace('\u0cc6\u0cc2','\u0cca')
+        text=text.replace('\u0cca\u0cd5','\u0ccb')
+
+        # correct visarge 
+        text=re.sub(r'([\u0c80-\u0cff]):','\\1\u0c83',text)
+
+        return text
+
+
+class MalayalamNormalizer(BaseNormalizer): 
+    """
+    Normalizer for the Malayalam script. In addition to basic normalization by the super class, 
+
+    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama 
+    * canonicalize two-part dependent vowel signs
+    * Change from old encoding of chillus (till Unicode 5.0) to new encoding
+    * replace colon ':' by visarga if the colon follows a charcter in this script 
+    """
+
+    CHILLU_CHAR_MAP= {
+                    '\u0d7a': '\u0d23', 
+                    '\u0d7b': '\u0d28',
+                    '\u0d7c': '\u0d30',
+                    '\u0d7d': '\u0d32',
+                    '\u0d7e': '\u0d33',
+                    '\u0d7f': '\u0d15',
+                 }
+
+    def _canonicalize_chillus(self,text):
+        for chillu, char in MalayalamNormalizer.CHILLU_CHAR_MAP.items(): 
+            text=text.replace(chillu,'{}\u0d4d'.format(char)) 
+        return text
+
+    def _correct_geminated_T(self,text):
+        return text.replace('\u0d31\u0d4d\u0d31','\u0d1f\u0d4d\u0d1f')
+
+    def __init__(self,lang='ml',remove_nuktas=False,nasals_mode='do_nothing',do_normalize_chandras=False,
+                do_normalize_vowel_ending=False,
+                do_canonicalize_chillus=False, do_correct_geminated_T=False):
+        super(MalayalamNormalizer,self).__init__(lang,remove_nuktas,nasals_mode,do_normalize_chandras,do_normalize_vowel_ending)
+        self.do_canonicalize_chillus=do_canonicalize_chillus
+        self.do_correct_geminated_T=do_correct_geminated_T
+
+    def normalize(self,text): 
+
+        # Change from old encoding of chillus (till Unicode 5.0) to new encoding
+        text=text.replace('\u0d23\u0d4d\u200d','\u0d7a')
+        text=text.replace('\u0d28\u0d4d\u200d','\u0d7b')
+        text=text.replace('\u0d30\u0d4d\u200d','\u0d7c')
+        text=text.replace('\u0d32\u0d4d\u200d','\u0d7d')
+        text=text.replace('\u0d33\u0d4d\u200d','\u0d7e')
+        text=text.replace('\u0d15\u0d4d\u200d','\u0d7f')
+
+        # Normalize chillus
+        if self.do_canonicalize_chillus:
+            text=self._canonicalize_chillus(text)
+
+        # common normalization for Indic scripts 
+        text=super(MalayalamNormalizer,self).normalize(text)
+
+        # replace the poorna virama codes specific to script 
+        # with generic Indic script codes
+        text=text.replace('\u0d64','\u0964')
+        text=text.replace('\u0d65','\u0965')
+
+        # dependent vowels
+        text=text.replace('\u0d46\u0d3e','\u0d4a')
+        text=text.replace('\u0d47\u0d3e','\u0d4b')
+
+        # au forms
+        text=text.replace('\u0d46\u0d57','\u0d4c')
+        text=text.replace('\u0d57','\u0d4c')
+
+        # correct geminated T
+        if self.do_correct_geminated_T:
+            text=self._correct_geminated_T(text)
+
+        # correct visarga 
+        text=re.sub(r'([\u0d00-\u0d7f]):','\\1\u0d03',text)
+
+        return text
+
+class UrduNormalizer(NormalizerI):
+    '''Uses UrduHack library.
+    https://docs.urduhack.com/en/stable/_modules/urduhack/normalization/character.html#normalize
+    '''
+
+    def __init__(self, lang, remove_nuktas=True):
+        self.lang = lang
+        self.remove_nuktas = remove_nuktas
+    
+        from urduhack.normalization import (
+            remove_diacritics,
+            normalize_characters,
+            normalize_combine_characters
+        ) # TODO: Use only required normalizers
+        from urduhack.preprocessing import (
+            normalize_whitespace,
+            digits_space,
+            all_punctuations_space,
+            english_characters_space
+        )
+
+    def normalize(self, text):
+        text = self._normalize_punctuations(text)
+        text = UrduNormalizer.normalize_whitespace(text)
+        if self.remove_nuktas:
+            text = UrduNormalizer.remove_diacritics(text)
+        text = UrduNormalizer.normalize_characters(text)
+        text = UrduNormalizer.normalize_combine_characters(text)
+        text = UrduNormalizer.digits_space(text)
+        text = UrduNormalizer.all_punctuations_space(text)
+        text = UrduNormalizer.english_characters_space(text)
+        return text
+
+
+class IndicNormalizerFactory(object):
+    """
+    Factory class to create language specific normalizers. 
+
+    """
+
+    def get_normalizer(self,language,**kwargs):
+        """
+            Call the get_normalizer function to get the language specific normalizer
+
+            Paramters: 
+            |language: language code
+            |remove_nuktas: boolean, should the normalizer remove nukta characters 
+        """
+        normalizer=None
+        if language in ['hi','mr','sa','kK','ne','sd']:
+            normalizer=DevanagariNormalizer(lang=language, **kwargs)
+        elif language in ['ur']:
+            normalizer = UrduNormalizer(lang=language, **kwargs)
+        elif language in ['pa']:
+            normalizer=GurmukhiNormalizer(lang=language, **kwargs)
+        elif language in ['gu']:
+            normalizer=GujaratiNormalizer(lang=language, **kwargs)
+        elif language in ['bn']:
+            normalizer=BengaliNormalizer(lang=language, **kwargs)
+        elif language in ['as']:
+            normalizer=BengaliNormalizer(lang=language, **kwargs)
+        elif language in ['or']:
+            normalizer=OriyaNormalizer(lang=language, **kwargs)
+        elif language in ['ml']:
+            normalizer=MalayalamNormalizer(lang=language, **kwargs)
+        elif language in ['kn']:
+            normalizer=KannadaNormalizer(lang=language, **kwargs)
+        elif language in ['ta']:
+            normalizer=TamilNormalizer(lang=language, **kwargs)
+        elif language in ['te']:
+            normalizer=TeluguNormalizer(lang=language, **kwargs)
+        else:    
+            normalizer=BaseNormalizer(lang=language, **kwargs)
+
+        return normalizer    
+
+    def is_language_supported(self,language): 
+        """
+        Is the language supported?
+        """
+        if language in ['hi','mr','sa','kK','ne','sd',
+                        'ur',
+                        'pa',
+                        'gu',
+                        'bn','as',
+                        'or',
+                        'ml',
+                        'kn',
+                        'ta',
+                        'te']:
+            return True
+        else:
+            return False
+
+
+if __name__ == '__main__': 
+
+    if len(sys.argv)<4:
+        print("Usage: python normalize.py <infile> <outfile> <language> [<replace_nukta(True,False)>] [<normalize_nasals(do_nothing|to_anusvaara_strict|to_anusvaara_relaxed|to_nasal_consonants)>]") 
+        sys.exit(1)
+
+    language=sys.argv[3]
+    remove_nuktas=False
+    normalize_nasals='do_nothing'
+    if len(sys.argv)>=5:
+        remove_nuktas=bool(sys.argv[4])
+    if len(sys.argv)>=6:
+        normalize_nasals=sys.argv[5]
+
+    # create normalizer
+    factory=IndicNormalizerFactory()
+    normalizer=factory.get_normalizer(language,remove_nuktas=remove_nuktas,nasals_mode=normalize_nasals)
+
+    # DO normalization 
+    with codecs.open(sys.argv[1],'r','utf-8') as ifile:
+        with codecs.open(sys.argv[2],'w','utf-8') as ofile:
+            for line in ifile.readlines():
+                normalized_line=normalizer.normalize(line)
+                ofile.write(normalized_line)
+   
+    ## gather status about normalization 
+    #with codecs.open(sys.argv[1],'r','utf-8') as ifile:
+    #    normalizer=DevanagariNormalizer()
+    #    text=string.join(ifile.readlines(),sep='')
+    #    normalizer.get_char_stats(text)

indic_nlp_library/indicnlp/script/english_script.py

@@ -0,0 +1,154 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import pandas as pd
+import numpy as np
+
+from indicnlp import common
+from indicnlp.common import IndicNlpException
+
+
+#### Maps from ARPABET to Internal Id
+ARPABET_ID_MAP={}
+ID_ARPABET_MAP={}
+
+
+###
+# Phonetic Information about script characters 
+###
+
+""" Phonetic data for English """
+ENGLISH_PHONETIC_DATA=None
+
+""" Phonetic vector for English"""
+ENGLISH_PHONETIC_VECTORS=None
+
+""" Length of phonetic vector """
+PHONETIC_VECTOR_LENGTH=38
+
+""" Start offset for the phonetic feature vector in the phonetic data vector """
+PHONETIC_VECTOR_START_OFFSET=6
+
+## PHONETIC PROPERTIES in order in which they occur in the vector 
+## This list must be in sync with the keys in the PV_PROP_RANGES dictionary 
+PV_PROP=['basic_type',
+    'vowel_length',
+    'vowel_strength',
+    'vowel_status',
+    'consonant_type',
+    'articulation_place',
+    'aspiration',
+    'voicing',
+    'nasalization',
+    'vowel_horizontal',
+    'vowel_vertical',
+    'vowel_roundness',
+    ]
+
+### 
+# Bit vector ranges for various properties 
+###
+
+PV_PROP_RANGES={
+        'basic_type': [0,6],
+        'vowel_length': [6,8],
+        'vowel_strength': [8,11],
+        'vowel_status': [11,13],
+        'consonant_type': [13,18],
+        'articulation_place': [18,23],
+        'aspiration': [23,25],
+        'voicing': [25,27],
+        'nasalization': [27,29],
+        'vowel_horizontal': [29,32],
+        'vowel_vertical': [32,36],
+        'vowel_roundness': [36,38],
+        }
+
+
+####
+# Indexes into the Phonetic Vector 
+####
+PVIDX_BT_VOWEL=0
+PVIDX_BT_CONSONANT=1
+PVIDX_BT_NUKTA=2
+PVIDX_BT_HALANT=3
+PVIDX_BT_ANUSVAAR=4
+PVIDX_BT_MISC=5
+PVIDX_BT_S=PVIDX_BT_VOWEL 
+PVIDX_BT_E=PVIDX_BT_MISC+1
+
+PVIDX_VSTAT_DEP=12
+
+####
+SCRIPT_RANGE_START=0x0D00
+## TBD
+SCRIPT_RANGE_END=0x0D2E
+
+
+def init():
+    """
+    To be called by library loader, do not call it in your program 
+    """
+
+    global ENGLISH_PHONETIC_DATA, ENGLISH_PHONETIC_VECTORS, PHONETIC_VECTOR_LENGTH, PHONETIC_VECTOR_START_OFFSET
+
+    ENGLISH_PHONETIC_DATA=pd.read_csv(common.get_resources_path()+'/script/english_script_phonetic_data.csv',encoding='utf-8')    
+
+    ENGLISH_PHONETIC_VECTORS=ENGLISH_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
+
+    PHONETIC_VECTOR_LENGTH=ENGLISH_PHONETIC_VECTORS.shape[1]
+
+    ### Load mapping from ARPABET representation of phoneme to internal ID
+    global ARPABET_ID_MAP, ID_ARPABET_MAP
+
+    with open(common.get_resources_path()+'/script/english_arpabet_list.csv','r',encoding='utf-8') as infile: 
+        for ph_id, name in enumerate(iter(infile)): 
+            name=name.strip()
+            ARPABET_ID_MAP[name]=ph_id
+            ID_ARPABET_MAP[ph_id]=name
+
+
+def phoneme_to_offset(ph): 
+    return ARPABET_ID_MAP[ph]
+
+def offset_to_phoneme(ph_id): 
+    return ID_ARPABET_MAP[ph_id]
+
+def phoneme_to_enc(ph): 
+    return chr(SCRIPT_RANGE_START+phoneme_to_offset(ph))
+
+def enc_to_phoneme(ph): 
+    return offset_to_phoneme(enc_to_offset(ph))
+
+def enc_to_offset(c): 
+    return ord(c)-SCRIPT_RANGE_START
+
+def in_range(offset): 
+    return offset>=SCRIPT_RANGE_START and offset<SCRIPT_RANGE_END 
+
+def get_phonetic_info(lang): 
+    return (ENGLISH_PHONETIC_DATA, ENGLISH_PHONETIC_VECTORS)
+
+def invalid_vector():
+    ##  TODO: check if np datatype is correct?
+    return np.array([0]*PHONETIC_VECTOR_LENGTH)
+
+def get_phonetic_feature_vector(p,lang):
+
+    offset=enc_to_offset(p) 
+
+    if not in_range(offset): 
+        return invalid_vector()
+
+    phonetic_data, phonetic_vectors= get_phonetic_info(lang)
+
+    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
+        return invalid_vector()
+
+    return phonetic_vectors[offset]
+

indic_nlp_library/indicnlp/script/indic_scripts.py

@@ -0,0 +1,301 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import pandas as pd
+import numpy as np
+import os
+
+from indicnlp import common
+from indicnlp.common import IndicNlpException
+from indicnlp import langinfo as li
+
+###
+# Phonetic Information about script characters 
+###
+
+""" Phonetic data about all languages except Tamil """
+ALL_PHONETIC_DATA=None
+
+""" Phonetic data for Tamil """
+TAMIL_PHONETIC_DATA=None
+
+""" Phonetic vector for all languages except Tamil """
+ALL_PHONETIC_VECTORS=None
+
+""" Phonetic vector for Tamil """
+TAMIL_PHONETIC_VECTORS=None
+
+""" Length of phonetic vector """
+PHONETIC_VECTOR_LENGTH=38
+
+""" Start offset for the phonetic feature vector in the phonetic data vector """
+PHONETIC_VECTOR_START_OFFSET=6
+
+## PHONETIC PROPERTIES in order in which they occur in the vector 
+## This list must be in sync with the keys in the PV_PROP_RANGES dictionary 
+PV_PROP=['basic_type',
+    'vowel_length',
+    'vowel_strength',
+    'vowel_status',
+    'consonant_type',
+    'articulation_place',
+    'aspiration',
+    'voicing',
+    'nasalization',
+    'vowel_horizontal',
+    'vowel_vertical',
+    'vowel_roundness',
+    ]
+
+### 
+# Bit vector ranges for various properties 
+###
+
+PV_PROP_RANGES={
+        'basic_type': [0,6],
+        'vowel_length': [6,8],
+        'vowel_strength': [8,11],
+        'vowel_status': [11,13],
+        'consonant_type': [13,18],
+        'articulation_place': [18,23],
+        'aspiration': [23,25],
+        'voicing': [25,27],
+        'nasalization': [27,29],
+        'vowel_horizontal': [29,32],
+        'vowel_vertical': [32,36],
+        'vowel_roundness': [36,38],
+        }
+
+
+####
+# Indexes into the Phonetic Vector 
+####
+PVIDX_BT_VOWEL=0
+PVIDX_BT_CONSONANT=1
+PVIDX_BT_NUKTA=2
+PVIDX_BT_HALANT=3
+PVIDX_BT_ANUSVAAR=4
+PVIDX_BT_MISC=5
+PVIDX_BT_S=PVIDX_BT_VOWEL 
+PVIDX_BT_E=PVIDX_BT_MISC+1
+
+PVIDX_VSTAT_DEP=12
+
+#####
+# Unicode information about characters 
+#####
+
+SCRIPT_OFFSET_START=0
+SCRIPT_OFFSET_RANGE=0x80
+
+def init():
+    """
+    To be called by library loader, do not call it in your program 
+    """
+
+    global ALL_PHONETIC_DATA, ALL_PHONETIC_VECTORS, TAMIL_PHONETIC_DATA, TAMIL_PHONETIC_VECTORS, PHONETIC_VECTOR_LENGTH, PHONETIC_VECTOR_START_OFFSET
+
+    ALL_PHONETIC_DATA=pd.read_csv(os.path.join(common.get_resources_path(),'script','all_script_phonetic_data.csv'),encoding='utf-8')    
+    TAMIL_PHONETIC_DATA=pd.read_csv(os.path.join(common.get_resources_path(),'script','tamil_script_phonetic_data.csv'),encoding='utf-8')    
+
+    ALL_PHONETIC_VECTORS= ALL_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
+    TAMIL_PHONETIC_VECTORS=TAMIL_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
+
+    PHONETIC_VECTOR_LENGTH=ALL_PHONETIC_VECTORS.shape[1]
+
+def is_supported_language(lang): 
+    return lang in list(li.SCRIPT_RANGES.keys())
+
+def get_offset(c,lang): 
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    return ord(c)-li.SCRIPT_RANGES[lang][0]
+
+def offset_to_char(off,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    """
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    return chr(off+li.SCRIPT_RANGES[lang][0])
+
+def is_indiclang_char(c,lang): 
+    """
+    Applicable to Brahmi derived Indic scripts 
+    Note that DANDA and DOUBLE_DANDA have the same Unicode codepoint for all Indic scripts 
+    """
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    o=get_offset(c,lang)
+    return (o>=SCRIPT_OFFSET_START and o<SCRIPT_OFFSET_RANGE) \
+            or ord(c)==li.DANDA or ord(c)==li.DOUBLE_DANDA
+
+def in_coordinated_range_offset(c_offset): 
+    """ 
+    Applicable to Brahmi derived Indic scripts 
+    """
+    return  (c_offset>=li.COORDINATED_RANGE_START_INCLUSIVE and c_offset<=li.COORDINATED_RANGE_END_INCLUSIVE) 
+
+def in_coordinated_range(c,lang):
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    return in_coordinated_range_offset(get_offset(c,lang))
+
+def get_phonetic_info(lang): 
+    if not is_supported_language(lang): 
+        raise IndicNlpException('Language {}  not supported'.format(lang))
+    phonetic_data= ALL_PHONETIC_DATA if lang!=li.LC_TA else TAMIL_PHONETIC_DATA
+    phonetic_vectors= ALL_PHONETIC_VECTORS if lang!=li.LC_TA else TAMIL_PHONETIC_VECTORS
+   
+    return (phonetic_data, phonetic_vectors)
+
+def invalid_vector():
+    ##  TODO: check if np datatype is correct?
+    return np.array([0]*PHONETIC_VECTOR_LENGTH)
+
+def get_phonetic_feature_vector(c,lang):
+
+    offset=get_offset(c,lang) 
+
+    if not in_coordinated_range_offset(offset): 
+        return invalid_vector()
+
+    phonetic_data, phonetic_vectors= get_phonetic_info(lang)
+
+    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
+        return invalid_vector()
+
+    return phonetic_vectors[offset]
+
+def get_phonetic_feature_vector_offset(offset,lang):
+
+    if not in_coordinated_range_offset(offset): 
+        return invalid_vector()
+
+    phonetic_data, phonetic_vectors= get_phonetic_info(lang)
+
+    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
+        return invalid_vector()
+
+    return phonetic_vectors[offset]
+
+### Unary operations on vectors 
+def is_valid(v): 
+    return np.sum(v)>0
+
+def is_vowel(v): 
+    return v[PVIDX_BT_VOWEL]==1 
+
+def is_consonant(v): 
+    return  v[PVIDX_BT_CONSONANT]==1 
+
+def is_halant(v): 
+    return  v[PVIDX_BT_HALANT]==1 
+
+def is_nukta(v): 
+    return  v[PVIDX_BT_NUKTA]==1 
+
+def is_anusvaar(v): 
+    return  v[PVIDX_BT_ANUSVAAR]==1 
+
+def is_misc(v): 
+    return v[PVIDX_BT_MISC]==1 
+
+def is_dependent_vowel(v): 
+    return is_vowel(v) and v[PVIDX_VSTAT_DEP]==1
+
+def is_plosive(v): 
+    return is_consonant(v) and get_property_vector(v,'consonant_type')[0]==1
+
+### Binary operations on phonetic vectors
+
+def or_vectors(v1,v2): 
+    return np.array([ 1 if (b1+b2)>=1 else 0 for b1,b2 in zip(v1,v2) ])
+
+def xor_vectors(v1,v2): 
+    return np.array([ 1 if b1!=b2 else 0 for b1,b2 in zip(v1,v2) ])
+
+### Getting properties from phonetic vectors 
+
+def get_property_vector(v,prop_name): 
+    return v[PV_PROP_RANGES[prop_name][0]:PV_PROP_RANGES[prop_name][1]]
+
+def get_property_value(v,prop_name): 
+    factor_bits=get_property_vector(v,prop_name).tolist()
+    
+    v=0
+    c=1
+    for b in factor_bits[::-1]: 
+        v+=(c*b)
+        c=c*2.0
+
+    return int(v)
+
+def lcsr_indic(srcw,tgtw,slang,tlang):
+    """
+    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.
+    This works for Indic scripts by mapping both languages to a common script
+
+    srcw: source language string
+    tgtw: source language string
+    slang: source language 
+    tlang: target language 
+    """    
+    score_mat=np.zeros((len(srcw)+1,len(tgtw)+1))
+
+    for si,sc in enumerate(srcw,1): 
+        for ti,tc in enumerate(tgtw,1): 
+            so=get_offset(sc,slang)
+            to=get_offset(tc,tlang)
+
+            if in_coordinated_range_offset(so) and in_coordinated_range_offset(to) and so==to: 
+                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
+            elif not (in_coordinated_range_offset(so) or in_coordinated_range_offset(to)) and sc==tc: 
+                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
+            else: 
+                score_mat[si,ti]= max(
+                    score_mat[si,ti-1],
+                    score_mat[si-1,ti])
+
+    return (score_mat[-1,-1]/float(max(len(srcw),len(tgtw))),float(len(srcw)),float(len(tgtw)))
+
+def lcsr_any(srcw,tgtw):
+    """
+    LCSR computation if both languages have the same script
+    """
+    score_mat=np.zeros((len(srcw)+1,len(tgtw)+1))
+
+    for si,sc in enumerate(srcw,1): 
+        for ti,tc in enumerate(tgtw,1): 
+
+            if sc==tc: 
+                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
+            else: 
+                score_mat[si,ti]= max(
+                    score_mat[si,ti-1],
+                    score_mat[si-1,ti])
+
+    return (score_mat[-1,-1]/float(max(len(srcw),len(tgtw))),float(len(srcw)),float(len(tgtw)))
+
+def lcsr(srcw,tgtw,slang,tlang):
+    """
+    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.
+
+    srcw: source language string
+    tgtw: source language string
+    slang: source language 
+    tlang: target language 
+    """
+
+    if slang==tlang or not is_supported_language(slang) or not is_supported_language(tlang):
+        return lcsr_any(srcw,tgtw,slang,tlang)
+    else:  
+        return lcsr_indic(srcw,tgtw)
+
+
+

indic_nlp_library/indicnlp/script/phonetic_sim.py

@@ -0,0 +1,59 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+from indicnlp import loader
+from indicnlp import langinfo
+from indicnlp.script.indic_scripts import * 
+import numpy as np
+import gzip
+import pandas as pd
+import sys 
+
+def equal(v1,v2): 
+    return 0.0 if  np.sum( xor_vectors(v1, v2)) > 0  else 1.0
+
+def dice(v1,v2):
+    dotprod=2*float(np.dot( v1, v2.T ))
+    return dotprod/float(len(v1)+len(v2))
+
+def jaccard(v1,v2):
+    dotprod=float(np.dot( v1, v2.T ))
+    return dotprod/float(len(v1)+len(v2)-dotprod)
+
+def cosine(v1,v2):
+    dotprod=float(np.dot( v1, v2.T ))
+    norm1=float(np.dot( v1, v1.T ))
+    norm2=float(np.dot( v2, v2.T ))
+    return ((dotprod)/(np.sqrt(norm1*norm2)+0.00001))
+
+def dotprod(v1,v2): 
+    return float(np.dot( v1, v2.T ))
+
+def sim1(v1,v2,base=5.0): 
+    return np.power(base,dotprod(v1,v2)) 
+
+def softmax(v1,v2): 
+    return sim1(v1,v2,np.e)
+
+def create_similarity_matrix(sim_func,slang,tlang,normalize=True):
+
+    dim=langinfo.COORDINATED_RANGE_END_INCLUSIVE-langinfo.COORDINATED_RANGE_START_INCLUSIVE+1    
+    sim_mat=np.zeros((dim,dim))    
+
+    for offset1 in range(langinfo.COORDINATED_RANGE_START_INCLUSIVE, langinfo.COORDINATED_RANGE_END_INCLUSIVE+1): 
+        v1=get_phonetic_feature_vector(offset_to_char(offset1,slang),slang)
+        for offset2 in range(langinfo.COORDINATED_RANGE_START_INCLUSIVE, langinfo.COORDINATED_RANGE_END_INCLUSIVE+1): 
+            v2=get_phonetic_feature_vector(offset_to_char(offset2,tlang),tlang)
+            sim_mat[offset1,offset2]=sim_func(v1,v2)
+
+    if normalize: 
+        sums=np.sum(sim_mat, axis=1)
+        sim_mat=(sim_mat.transpose()/sums).transpose()
+
+    return sim_mat
+

indic_nlp_library/indicnlp/syllable/syllabifier.py

@@ -0,0 +1,302 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+import codecs, sys
+from indicnlp.script import indic_scripts as si 
+import re
+
+chillu_char_map= {
+                    '\u0d7a': '\u0d23', 
+                    '\u0d7b': '\u0d28',
+                    '\u0d7c': '\u0d30',
+                    '\u0d7d': '\u0d32',
+                    '\u0d7e': '\u0d33',
+                    '\u0d7f': '\u0d15',
+                 }
+
+char_chillu_map= {}
+for k,v in chillu_char_map.items():
+    char_chillu_map[v]=k
+
+def normalize_malayalam(word): 
+
+    word_mask=re.sub(r'[0-9]','0',word)
+
+    # instead of chillu characters, use consonant+halant 
+    for chillu,char in chillu_char_map.items(): 
+        word=word.replace(chillu,'{}\u0d4d'.format(char)) 
+        word_mask=word_mask.replace(chillu,'41')
+
+    word_mask=re.sub(r'[^0-9]','0',word_mask)
+
+    return word, word_mask
+
+def denormalize_malayalam(word, word_mask):
+
+    word=list(word)
+    word_mask=list(word_mask)
+
+    ## pattern 4
+    idx=0
+    while idx>=0: 
+        try: 
+            idx=word_mask.index('4',idx)
+            word[idx:idx+2]=char_chillu_map[word[idx]]
+            word_mask[idx:idx+2]='0'
+            start=idx            
+        except ValueError as e: 
+            break
+
+    return ''.join(word)
+
+def normalize_punjabi(word): 
+    word_mask=re.sub(r'[0-9]','0',word)
+
+    ## replace tippi with anusvaar
+    word=word.replace('\u0a70','\u0a02')
+    word_mask=word_mask.replace('\u0a70','2')
+
+    ## replace addak+consonant with consonat+halant+consonant
+    word=re.sub(r'\u0a71(.)','\\1\u0a4d\\1',word)
+    word_mask=re.sub(r'\u0a71(.)','311',word_mask)
+
+    word_mask=re.sub(r'[^0-9]','0',word_mask)
+
+    return word, word_mask
+
+def denormalize_punjabi(word, word_mask):
+    
+    word=list(word)
+    word_mask=list(word_mask)
+
+    ## pattern 2
+    idx=0
+    while idx>=0: 
+        try: 
+            idx=word_mask.index('2',idx)
+            word[idx]='\u0a70'
+            word_mask[idx]='0'
+            start=idx            
+        except ValueError as e: 
+            break
+
+    ## pattern 3
+    idx=0
+    while idx>=0: 
+        try: 
+            idx=word_mask.index('3',idx)
+            word[idx:idx+3]='\u0a71{}'.format(word[idx])
+            word_mask[idx:idx+3]='00'
+            start=idx            
+        except ValueError as e: 
+            break
+
+    return ''.join(word)
+
+def char_backoff(syllables_list,vocab):
+    syllables_final=[]    
+
+    if vocab is None:
+        syllables_final=syllables_list
+    else:
+        for s in syllables_list:
+            if s in vocab: 
+                syllables_final.append(s)
+            else: 
+                for x in s:
+                    syllables_final.append(x)                    
+
+    return syllables_final        
+
+
+def orthographic_syllabify_improved(word,lang,vocab=None): 
+
+    word_mask=['0']*len(word)
+
+    if lang=='ml':
+        word, word_mask = normalize_malayalam(word)
+        word=word
+    elif lang=='pa':
+        word, word_mask = normalize_punjabi(word)
+
+    p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word]
+
+    syllables=[]
+    syllables_mask=[]
+
+    for i in range(len(word)): 
+        v=p_vectors[i]
+
+        syllables.append(word[i])
+        syllables_mask.append(word_mask[i])
+
+        ### simplified syllabification 
+        #if i+1<len(word) and \
+        #        (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+        #    syllables.append(u' ')
+        #    syllables_mask.append(u'0')
+
+        #elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v):
+        #    syllables.append(u' ')
+        #    syllables_mask.append(u'0')
+
+        #elif i+1<len(word) and \
+        #     (si.is_consonant(v) or si.is_nukta(v)) and \
+        #     (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])):
+        #    syllables.append(u' ')
+        #    syllables_mask.append(u'0')
+
+        #### better syllabification 
+        if i+1<len(word) and (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+            syllables.append(' ')
+            syllables_mask.append('0')
+
+        elif not si.is_valid(v) or si.is_misc(v) :
+            syllables.append(' ')
+            syllables_mask.append('0')
+
+        elif si.is_vowel(v):
+
+            anu_nonplos= ( i+2<len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) and \
+                           not si.is_plosive(p_vectors[i+2])\
+                         )
+            
+            anu_eow= ( i+2==len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) )
+
+            if not(anu_nonplos or anu_eow):              
+                syllables.append(' ')
+                syllables_mask.append('0')
+
+        elif i+1<len(word) and \
+                (si.is_consonant(v) or si.is_nukta(v)): 
+            if si.is_consonant(p_vectors[i+1]): 
+                syllables.append(' ')
+                syllables_mask.append('0')
+            elif si.is_vowel(p_vectors[i+1]) and \
+                    not si.is_dependent_vowel(p_vectors[i+1]): 
+                syllables.append(' ')
+                syllables_mask.append('0')
+            elif si.is_anusvaar(p_vectors[i+1]):
+                anu_nonplos= ( i+2<len(word) and \
+                               not si.is_plosive(p_vectors[i+2])\
+                             )
+                
+                anu_eow= i+2==len(word) 
+
+                if not(anu_nonplos or anu_eow):              
+                    syllables.append(' ')
+                    syllables_mask.append('0')
+
+    syllables_mask=''.join(syllables_mask)
+    syllables=''.join(syllables)
+
+    #assert len(syllables_mask) == len(syllables)
+    #assert syllables_mask.find('01') == -1
+    if syllables_mask.find('01') >= 0: 
+        print('Warning')
+
+    if lang=='ml': 
+        syllables = denormalize_malayalam(syllables,syllables_mask)
+    elif lang=='pa': 
+        syllables = denormalize_punjabi(syllables,syllables_mask)
+
+    syllables_list = syllables.strip().split(' ')  
+    return(char_backoff(syllables_list,vocab))    
+
+def orthographic_syllabify(word,lang,vocab=None): 
+
+    p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word]
+
+    syllables=[]
+
+    for i in range(len(word)): 
+        v=p_vectors[i]
+
+        syllables.append(word[i])
+
+        ### simplified syllabification 
+        #if i+1<len(word) and \
+        #        (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+        #    syllables.append(u' ')
+
+        #elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v):
+        #    syllables.append(u' ')
+
+        #elif i+1<len(word) and \
+        #     (si.is_consonant(v) or si.is_nukta(v)) and \
+        #     (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])):
+        #    syllables.append(u' ')
+
+        #### better syllabification 
+        if i+1<len(word) and (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+            syllables.append(' ')
+
+        elif not si.is_valid(v) or si.is_misc(v) :
+            syllables.append(' ')
+
+        elif si.is_vowel(v):
+
+            anu_nonplos= ( i+2<len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) and \
+                           not si.is_plosive(p_vectors[i+2])\
+                         )
+            
+            anu_eow= ( i+2==len(word) and \
+                           si.is_anusvaar(p_vectors[i+1]) )
+
+            if not(anu_nonplos or anu_eow):              
+                syllables.append(' ')
+
+        elif i+1<len(word) and \
+                (si.is_consonant(v) or si.is_nukta(v)): 
+            if si.is_consonant(p_vectors[i+1]): 
+                syllables.append(' ')
+            elif si.is_vowel(p_vectors[i+1]) and \
+                    not si.is_dependent_vowel(p_vectors[i+1]): 
+                syllables.append(' ')
+            elif si.is_anusvaar(p_vectors[i+1]):
+                anu_nonplos= ( i+2<len(word) and \
+                               not si.is_plosive(p_vectors[i+2])\
+                             )
+                
+                anu_eow= i+2==len(word) 
+
+                if not(anu_nonplos or anu_eow):              
+                    syllables.append(' ')
+
+    syllables_list = ''.join(syllables).strip().split(' ') 
+    return(char_backoff(syllables_list,vocab))
+
+def orthographic_simple_syllabify(word,lang,vocab=None): 
+
+    p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word]
+
+    syllables=[]
+
+    for i in range(len(word)): 
+        v=p_vectors[i]
+
+        syllables.append(word[i])
+
+        ## simplified syllabification 
+        if i+1<len(word) and \
+                (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])):
+            syllables.append(' ')
+
+        elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v):
+            syllables.append(' ')
+
+        elif i+1<len(word) and \
+             (si.is_consonant(v) or si.is_nukta(v)) and \
+             (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])):
+            syllables.append(' ')
+
+    syllables_list = ''.join(syllables).strip().split(' ') 
+    return(char_backoff(syllables_list,vocab))

indic_nlp_library/indicnlp/tokenize/indic_detokenize.py

@@ -0,0 +1,134 @@
+# 
+#  Copyright (c) 2013-present, Anoop Kunchukuttan
+#  All rights reserved.
+#  
+#  This source code is licensed under the MIT license found in the
+#  LICENSE file in the root directory of this source tree.
+# 
+
+#Program for detokenizing Indian language input 
+#
+# @author Anoop Kunchukuttan 
+#
+"""
+De-tokenizer for Indian languages.
+"""
+
+import string, re, sys
+from indicnlp.common import IndicNlpException
+
+## detokenizer patterns 
+left_attach=r'!%)\]},.:;>?\u0964\u0965'
+pat_la=re.compile(r'[ ](['+left_attach+r'])')
+
+right_attach=r'#$(\[{<@'
+pat_ra=re.compile(r'(['+right_attach+r'])[ ]')
+
+lr_attach=r'-/\\'
+pat_lra=re.compile(r'[ ](['+lr_attach+r'])[ ]')
+
+#donknow=u'&*+=^_|~'
+
+## date, numbers, section/article numbering
+## TODO: handle indic numbers
+pat_num_seq=re.compile(r'([0-9]+ [,.:/] )+[0-9]+')
+
+### e-mail address
+#pat_num=re.compile(ur'[a-zA-Z]+[ ]? 
+
+def trivial_detokenize_indic(text): 
+    """detokenize string for Indian language scripts using Brahmi-derived scripts
+
+    A trivial detokenizer which:
+
+        - decides whether punctuation attaches to left/right or both
+        - handles number sequences
+        - handles quotes smartly (deciding left or right attachment)
+
+    Args:
+        text (str): tokenized text to process 
+
+    Returns:
+        str: detokenized string
+    """
+
+    s=text
+    ### some normalizations 
+
+    #numbers and dates
+    new_s=''
+    prev=0
+    for m in pat_num_seq.finditer(s):
+        start=m.start()
+        end=m.end()
+        if start>prev:
+            new_s=new_s+s[prev:start]
+            new_s=new_s+s[start:end].replace(' ','')
+            prev=end
+   
+    new_s=new_s+s[prev:]
+    s=new_s
+
+    ###  consective single quotes or backslashes become double quotes
+    #s=s.replace("' '", "''")
+    #s=s.replace("` `", '``')
+
+    s=pat_lra.sub('\\1',s)
+    s=pat_la.sub('\\1',s)
+    s=pat_ra.sub('\\1',s)
+
+    # assumes well formedness of quotes and alternates between right and left attach
+
+    alt_attach='\'"`'
+    for punc in alt_attach: 
+        cnt=0
+        out_str=[]
+        for c in s:
+            if c == punc:
+                if cnt%2==0:
+                    out_str.append('@RA')
+                else:
+                    out_str.append('@LA')
+                cnt+=1    
+            else:
+                out_str.append(c)
+
+        s=''.join(out_str).replace('@RA ',punc).replace(' @LA',punc
+                ).replace('@RA',punc).replace('@LA',punc)
+
+    return s
+
+def trivial_detokenize(text,lang='hi'): 
+    """detokenize string for languages of the Indian subcontinent 
+
+    A trivial detokenizer which:
+
+        - decides whether punctuation attaches to left/right or both
+        - handles number sequences
+        - handles quotes smartly (deciding left or right attachment)
+
+    Args:
+        text (str): tokenized text to process 
+
+    Returns:
+        str: detokenized string
+
+    Raises:
+        IndicNlpException: If language is not supported        
+    """
+    if lang=='ur':
+        raise IndicNlpException('No detokenizer available for Urdu')
+    else:
+        return trivial_detokenize_indic(text)
+
+# if __name__ == '__main__': 
+
+#     if len(sys.argv)<4:
+#         print("Usage: python indic_detokenize.py <infile> <outfile> <language>")
+#         sys.exit(1)
+
+#     with open(sys.argv[1],'r', encoding='utf-8') as ifile:
+#         with open(sys.argv[2],'w', encoding='utf-8') as ofile:
+#             for line in ifile:
+#                 detokenized_line=trivial_detokenize(line,sys.argv[3])
+#                 ofile.write(detokenized_line)