responsible-ai-privacy/src/privacy/service/textPrivacy.py

'''
MIT license https://opensource.org/licenses/MIT Copyright 2024 Infosys Ltd

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''



# from privacy.dao.TelemetryFlagDb import TelemetryFlag
from privacy.mappers.mappers import *

from typing import List
from privacy.constants.local_constants import (DELTED_SUCCESS_MESSAGE)
from privacy.exception.exception import PrivacyNameNotEmptyError, PrivacyException, PrivacyNotFoundError

from privacy.config.logger import CustomLogger
log = CustomLogger()
from dotenv import load_dotenv
from privacy.config.logger import request_id_var
load_dotenv()
from faker import Faker
fake = Faker()

from privacy.util.special_recognizers.DataListRecognizer import DataListRecognizer
# global error_dict
from privacy.service.__init__ import *
from privacy.service.api_req import ApiCall

from privacy.util.special_recognizers.fakeData import FakeDataGenerate
from privacy.service.__init__ import *
from privacy.service.api_req import ApiCall


class TextPrivacy:
    def analyze(payload: PIIAnalyzeRequest) -> PIIAnalyzeResponse:
        error_dict[request_id_var.get()]=[]
        log.debug("Entering in analyze function")
        # gc.collect()
        log.debug(f"payload: {payload}")
        piiEntitiesToBeRedacted = payload.piiEntitiesToBeRedacted
        try:
            if(payload.exclusionList == None):
                exclusionList=[]
            else:
                exclusionList=payload.exclusionList

            if(payload.portfolio== None):
                results = TextPrivacy.textAnalyze(text=payload.inputText,piiEntitiesToBeRedacted=piiEntitiesToBeRedacted,exclusion=exclusionList)
            else:
                results = TextPrivacy.textAnalyze(text=payload.inputText,accName=payload,exclusion=exclusionList)
            if results == None:
                return None
            if( results== 404):
                return results
            if(results==482):
                return results
            list_PIIEntity = []
            results=sorted(results, key=lambda i: i.start)
            
            for result in results:
                log.debug(f"result: {result}")
                obj_PIIEntity = PIIEntity(type=result.entity_type,
                                          beginOffset=result.start,
                                          endOffset=result.end,
                                          score=result.score,
                                          responseText=payload.inputText[result.start:result.end])
                log.debug(f"obj_PIIEntity: {obj_PIIEntity}")
                list_PIIEntity.append(obj_PIIEntity)
                del obj_PIIEntity

            log.debug(f"list_PIIEntity: {list_PIIEntity}")
            objPIIAnalyzeResponse = PIIAnalyzeResponse
            objPIIAnalyzeResponse.PIIEntities = list_PIIEntity
            # gc.collect()
            log.debug("Returning from analyze function")
            return objPIIAnalyzeResponse
        except Exception as e:
            log.error(str(e))
            log.error("Line No:"+str(e.__traceback__.tb_lineno))
            log.error(str(e.__traceback__.tb_frame))
            error_dict[request_id_var.get()].append({"UUID":request_id_var.get(),"function":"textAnalyzeMainFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            # ExceptionDb.create({"UUID":request_id_var.get(),"function":"textAnalyzeMainFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            raise Exception(e)
    # @profile
    
    def textAnalyze(text: str,accName:any=None,exclusion:any=None,piiEntitiesToBeRedacted:any=None):
        result=[]
        
        
        try:
            if(accName==None):
                # print("result------",PIIAnalyzeRequest.language)
                # ent_pattern=[]
                if(piiEntitiesToBeRedacted == None):
                    result = analyzer.analyze(text=text,language="en",allow_list=exclusion,return_decision_process = True)
                    
                else:
                    try:
                        result = analyzer.analyze(text=text,language="en",entities=piiEntitiesToBeRedacted,allow_list=exclusion,return_decision_process = True)
                        #print("result------",result)
                    except Exception as e:
                        #print("error 103===",e)
                        return 482
                        # raise Exception("An error occurred while analyzing the text", e)

                # print("result------",result)
                # print("result====192====",result)

                        
            #     #score_threshold reference
            #     # gc.collect()
            #     print("ent_pattern list====",ent_pattern)

            else:
                preEntity=[]
                # entityType,datalist,preEntity=admin_par[request_id_var.get()]["scoreTreshold"].request(accName)
                # entityType,datalist,preEntity=ApiCall.request(accName)
                dataistEnt = []
                
                response_value=ApiCall.request(accName)

                if(response_value==None):
                    return None
                if(response_value==404):
                    # print( response_value)
                    return response_value
                
                entityType,datalist,preEntity=response_value
                #print('=====',datalist)
                for d in range(len(datalist)):
                    record=ApiCall.getRecord(entityType[d])
                    record=AttributeDict(record)
                    log.debug("Record====="+str(record))

                    # predefined_recognizers.data_recognizer.DataList.entity.clear()
                    # predefined_recognizers.data_recognizer.DataList.resetData()
                    if(record.RecogType=="Data"):
                            
                            dataRecog=(DataListRecognizer(terms=datalist[d],entitie=[entityType[d]]))
                            registry.add_recognizer(dataRecog)
                            # predefined_recognizers.data_recognizer.DataList.entity.append(entityType[d])
                            # predefined_recognizers.data_recognizer.DataList.setData(datalist[d])
                            #print("EntityTye226===",entityType[d])
                            #print("datalist===",datalist[d])
                            update_session_dict(entityType[d], datalist[d])
                            data = {entityType[d]: datalist[d]}
                            dataistEnt.append(data)
                 
                    elif(record.RecogType=="Pattern" and record.isPreDefined=="No"):
                        contextObj=record.Context.split(',')
                        pattern="|".join(datalist[d])
                        # print("=",pattern)
                        log.debug("pattern="+str(pattern))
                        patternObj = Pattern(name=entityType[d],
                                                       regex=pattern,
                                                       score=record.Score)
                        patternRecog = PatternRecognizer(supported_entity=entityType[d],
                                                                   patterns=[patternObj],context=contextObj)
                        registry.add_recognizer(patternRecog)

                    # result.clear()
                    # print(";;",entityType)
                #print("type 164dataistEnt====",type(entityType+preEntity))
                #print("164dataistEnt====",entityType+preEntity)
                results = analyzer.analyze(text=text, language="en",entities=entityType+preEntity,allow_list=exclusion,score_threshold=admin_par[request_id_var.get()]["scoreTreshold"])
                        # entityType.remove(preEntity)
                result.extend(results)
                    # preEntity.clear()
                # if len(preEntity) > 0:
                #     results = analyzer.analyze(text=text, language="en",entities=preEntity,allow_list=exclusion,score_threshold=admin_par[request_id_var.get()]["scoreTreshold"])
                #     preEntity.clear()
                #     result.extend(results)
                # predefined_recognizers.data_recognizer.DataList.entity.clear()
                # predefined_recognizers.data_recognizer.DataList.resetData()

                log.debug(f"results: {results}")
                log.debug(f"type results: {type(results)}")
                # result.extend(results)
            # gc.collect()
            # del analyzer
            # del registry
            # ApiCall.encryptionList.clear()
            log.debug("result="+str(result))

            return result
        except Exception as e:
            log.error(str(e))
            # print("======================",error_dict)
            log.error("Line No:"+str(e.__traceback__.tb_lineno))
            log.error(str(e.__traceback__.tb_frame))
            # ExceptionDb.create({"UUID":request_id_var.get(),"function":"textAnalyzeFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            error_dict[request_id_var.get()].append({"UUID":request_id_var.get(),"function":"textAnalyzeFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            raise Exception(e)
  
    def anonymize(payload: PIIAnonymizeRequest):
        error_dict[request_id_var.get()]=[]
        log.debug("Entering in anonymize function")
        try:
            # Data.encrypted_text.clear()  
            # print("list",registry)
            # print("list",registry.recognizers)
            # if(payload.piiEntitiesToBeRedacted != None):
            #     piiEntitiesToBeRedacted = payload.piiEntitiesToBeRedacted
            # else:
            piiEntitiesToBeRedacted = payload.piiEntitiesToBeRedacted 
            if(payload.exclusionList == None):
                exclusionList=[]
            else:
                exclusionList=payload.exclusionList
            if(payload.portfolio== None):
                results = TextPrivacy.textAnalyze(text=payload.inputText,exclusion=exclusionList,piiEntitiesToBeRedacted=piiEntitiesToBeRedacted)
                # print("results213====",type(results))
                
                # if(len(results)==2):
                #     print("results====",results[0])
                #     return results[0]
                
                
                    # obj_PIIAnonymizeResponse = PIIAnonymizeResponse
                    # print("results====",results[1])
                    # obj_PIIAnonymizeResponse.anonymizedText = str(results[1])
                    # return obj_PIIAnonymizeResponse
            else: 
                results = TextPrivacy.textAnalyze(text=payload.inputText,accName=payload,exclusion=exclusionList)
                
            ent_pattern=[]
            if results == None:
                return None

            if(results==404):
                    # print( response_value)
                    return results
                
            if(results==482):
                return results
            dict_operators = {} 
            if payload.fakeData == True:            
                #  fakeDataGeneration() used for generating fakeData for the entities whcih return dict containg the fake data is replaced with entity....
                fake_dict_operator = FakeDataGenerate.fakeDataGeneration(results,payload.inputText)
                dict_operators.update(fake_dict_operator)
        

            encryptionList=[]
            if(payload.portfolio!= None ):

                encryptionList=admin_par[request_id_var.get()]["encryptionList"]
            # print("==============================",encryptionList)
            if encryptionList is not None and len(encryptionList) >0 :
                for entity in encryptionList:
                    # print("---------------------------")
                    dict_operators.update({entity: OperatorConfig("hash", {"hash-type": 'md5'})})
            # else:
            #     dict_operators = None

            # ApiCall.encryptionList.clear()
            anonymize_text = anonymizer.anonymize(text=payload.inputText,
                                                  operators=dict_operators,
                                                  analyzer_results=results)


            log.debug(f"anonymize_text: {anonymize_text}")
            log.debug(f"anonymize_text_item"+ str(anonymize_text.items))

            obj_PIIAnonymizeResponse = PIIAnonymizeResponse
            obj_PIIAnonymizeResponse.anonymizedText = anonymize_text.text
            log.debug("Returning from anonymize function")

            return obj_PIIAnonymizeResponse
        except Exception as e:
            log.error(str(e))
            log.error("Line No:"+str(e.__traceback__.tb_lineno))
            log.error(str(e.__traceback__.tb_frame))
            error_dict[request_id_var.get()].append({"UUID":request_id_var.get(),"function":"textAnonimyzeFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            raise Exception(e)

    def encrypt(payload: PIIAnonymizeRequest):
        log.debug("Entering in encrypt function")
        
        try:
            # Data.encrypted_text.clear()  
            
            if(payload.exclusionList == None):
                exclusionList=[]
            else:
                exclusionList=payload.exclusionList
            if(payload.portfolio== None):
                results = TextPrivacy.textAnalyze(text=payload.inputText,exclusion=exclusionList)
            else: 
                results = TextPrivacy.textAnalyze(text=payload.inputText,accName=payload,exclusion=exclusionList)
                
            if results == None:
                return None
            dict_operators = {} 
            
            data = "WmZq4t7w!z%C&F)J"
                
            for i in results:
                dict_operators.update({i.entity_type : OperatorConfig("encrypt", {"key": data})})
            
            anonymize_text = anonymizer.anonymize(text=payload.inputText,
                                                  operators=dict_operators,
                                                  analyzer_results=results)
            
            log.debug(f"anonymize_text: {anonymize_text}")
            log.debug(f"anonymize_text_item"+ str(anonymize_text.items))
            
            obj_PIIEncryptResponse = PIIEncryptResponse
            obj_PIIEncryptResponse.text = anonymize_text.text
            obj_PIIEncryptResponse.items= anonymize_text.items
            log.debug("Returning from encrypt function")
            
            return obj_PIIEncryptResponse
        
        
        except Exception as e:
            log.error(str(e))
            log.error("Line No:"+str(e.__traceback__.tb_lineno))
            log.error(str(e.__traceback__.tb_frame))
            error_dict[request_id_var.get()].append({"UUID":request_id_var.get(),"function":"textAnonimyzeFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            raise Exception(e)
    
    def decryption(payload: PIIDecryptRequest):
        log.debug("Entering in decrypt function")
        # payload=AttributeDict(payload)
        print("payload=====",payload)
        try:
            anonymized_text = payload.text
            anonymized_entities = payload.items
            
            data = "WmZq4t7w!z%C&F)J"
            list_ent= []
            for item in anonymized_entities:
                list_ent.append(OperatorResult(start=item.start,
                                    end=item.end ,
                                    entity_type= item.entity_type,
                                    text= item.text,
                                    operator= item.operator,))
            anonymized_entities=list_ent    
            
            deanonymized_result = deanonymizer.deanonymize(text=anonymized_text,
                                            entities=anonymized_entities,
                                    operators={"DEFAULT": OperatorConfig("decrypt", {"key": data})},)
            
            obj_PIIDecryptResponse = PIIDecryptResponse
            obj_PIIDecryptResponse.decryptedText = deanonymized_result.text
            log.debug("Returning from anonymize function")

            return obj_PIIDecryptResponse
            
            
        except Exception as e:
            log.error(str(e))
            log.error("Line No:"+str(e.__traceback__.tb_lineno))
            log.error(str(e.__traceback__.tb_frame))
            error_dict[request_id_var.get()].append({"UUID":request_id_var.get(),"function":"textAnonimyzeFunction","msg":str(e.__class__.__name__),"description":str(e)+"Line No:"+str(e.__traceback__.tb_lineno)})
            raise Exception(e)
    
class Shield:
    def privacyShield(payload: PIIPrivacyShieldRequest) -> PIIPrivacyShieldResponse:
        log.debug("Entering in privacyShield function")
        log.debug(f"payload: {payload}")

        res = []
        totEnt=[]
        enres=[]
        query={}
       
        log.debug("response="+str(res))
        if(payload.portfolio== None):
            response_value=ApiCall.request(payload)
            if(response_value==None):
                return None
            if(response_value==404):
                    # print( response_value)
                    
                    return response_value
            entityType,datalist,preEntity=response_value
            # entityType,datalist,preEntity=ApiCall.request(payload) 
            results = TextPrivacy.textAnalyze(text=payload.inputText)
            # entity=RecogDb.findall({})
            record=[ele for ele in admin_par[request_id_var.get()]["records"] if ele["isPreDefined"]=="Yes"]
            
            for i in record:
                i=AttributeDict(i)
                totEnt.append(i.RecogName)
            pass
        else:
            # entityType,datalist,preEntity=ApiCall.request(payload) 
            response_value=ApiCall.request(payload)
            if(response_value==None):
                return None
            if(response_value==404):
                    # print( response_value)
                    return response_value
            entityType,datalist,preEntity=response_value
           
            to=[]
            # log.debug("entityTyope="+str(entityType))
            # log.debug("preEntity="+str(preEntity))
            entityType.extend(preEntity)
            # log.debug("entity="+str(entityType))
            totEnt=entityType
                
            results = TextPrivacy.textAnalyze(text=payload.inputText,accName=payload)
            # log.debug("total recoed="+str(totEnt))
        
        value=payload.inputText
        list_PIIEntity = []
        results=sorted(results, key=lambda i: i.start)
        for result in results:
            log.debug(f"result: {result}")
            enres.append({"type":result.entity_type,"start":result.start,"end":result.end,"value":value[result.start:result.end]})
            # obj_PIIEntity = PIIEntity(type=result.entity_type,
            #                           beginOffset=result.start,
            #                           endOffset=result.end)
            log.debug(f"obj_PIIEntity: {enres}")
            # list_PIIEntity.append(enres)
            # del obj_PIIEntity

        if(len(enres)==0):
            temp= "Passed"
        else:
            temp="Failed"

        objent = PrivacyShield(
             entitiesRecognised=enres,
             entitiesConfigured= totEnt,
             result=temp
        )
        list_PIIEntity.append(objent)
        log.debug(f"list_PIIEntity: {list_PIIEntity}")
        
        objPIIAnalyzeResponse = PIIPrivacyShieldResponse
        objPIIAnalyzeResponse.privacyCheck = list_PIIEntity


        log.debug("objPIIAnalyzeResponse="+str(objPIIAnalyzeResponse.privacyCheck))
        log.debug("Returning from privacyShield function")
        return objPIIAnalyzeResponse