Update app.py

app.py

@@ -1372,49 +1372,129 @@ def getLinearTextualContextFromTriples(word,labelTriplesLIST, text_splitter, arg
     if (strtobool(args.UseRetrieverForContextCreation)==True):
         labelTriples = ""
         passages = []
-        nn=200
-        for i, triple in enumerate(labelTriplesLIST, start=1):
-        #for triple in labelTriplesLIST:
-            TriplesString = (" ".join(str(element).capitalize() for element in triple))
-            passages.append(TriplesString)
-            # Check if the current index is a multiple of nn
-            if i % nn == 0:
-                #print("elaborate RAG triples")
-
-                #df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=50)
-                #df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0,  max_num_passages=50, min_threshold=0.65)
-                #df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90,  max_num_passages=50, min_threshold=0.65)
-                df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=50, min_threshold=0.65)
-
-                passages = []
-
-                df_retrieved = df_retrievedtopk.copy()
-                if not df_retrieved.empty:
-                    labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
-                    labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
-
-                    if not labelTriples:
-                        labelTriples =labelTriplesAPP
-                    else:
-                        labelTriples = labelTriples + ". " + labelTriplesAPP
+        nn = 200
+
+        if len(labelTriplesLIST)<=nn:
+            passages = []
+            for i, triple in enumerate(labelTriplesLIST, start=1):
+                # for triple in labelTriplesLIST:
+                TriplesString = (" ".join(str(element).capitalize() for element in triple))
+                passages.append(TriplesString)
 
-        if passages:
-            df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=50, min_threshold=0.65)
+            df_retrieved = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20,
+                                                  min_threshold=0.7)
 
-            df_retrieved = df_retrievedtopk.copy()
             if not df_retrieved.empty:
                 labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
-                labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
+                labelTriplesAPP = ". ".join(
+                    " ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
+
                 if not labelTriples:
                     labelTriples = labelTriplesAPP
                 else:
                     labelTriples = labelTriples + ". " + labelTriplesAPP
 
-        if labelTriples:
-            labelTriples.strip().replace("..",".").strip()
+        else:
 
+            OverallListRAGtriples = labelTriplesLIST.copy()
+
+            while len(OverallListRAGtriples)>nn:
+                Oinnerlistiterative=[]
+                for i, triple in enumerate(OverallListRAGtriples, start=1):
+                    # for triple in labelTriplesLIST:
+                    TriplesString = (" ".join(str(element).capitalize() for element in triple))
+                    passages.append(TriplesString)
+                    # Check if the current index is a multiple of nn
+                    if i % nn == 0:
+                        # print("elaborate RAG triples")
+
+                        # df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=20)
+                        # df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0,  max_num_passages=20, min_threshold=0.7)
+                        # df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90,  max_num_passages=20, min_threshold=0.7)
+                        df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20,
+                                                              min_threshold=0.7)
+
+                        passages = []
+
+                        df_retrieved = df_retrievedtopk.copy()
+                        if not df_retrieved.empty:
+                            labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
+                            if not Oinnerlistiterative:
+                                Oinnerlistiterative=labelTriplesLIST_RAGGED
+                            else:
+                                Oinnerlistiterative.extend(labelTriplesLIST_RAGGED)
+
+                if passages:
+                    df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20,
+                                                          min_threshold=0.7)
+
+                    df_retrieved = df_retrievedtopk.copy()
+                    if not df_retrieved.empty:
+                        labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
+                        if not Oinnerlistiterative:
+                            Oinnerlistiterative = labelTriplesLIST_RAGGED
+                        else:
+                            Oinnerlistiterative.extend(labelTriplesLIST_RAGGED)
+
+                OverallListRAGtriples = Oinnerlistiterative.copy()
+
+            if OverallListRAGtriples:
+                labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in OverallListRAGtriples)
 
-    else:
+                if not labelTriples:
+                    labelTriples = labelTriplesAPP
+                else:
+                    labelTriples = labelTriples + ". " + labelTriplesAPP
+
+                labelTriples = labelTriples.strip().replace("..", ".").strip()
+
+
+
+        # labelTriples = ""
+        # passages = []
+        # nn=200
+        # for i, triple in enumerate(labelTriplesLIST, start=1):
+        # #for triple in labelTriplesLIST:
+        #     TriplesString = (" ".join(str(element).capitalize() for element in triple))
+        #     passages.append(TriplesString)
+        #     # Check if the current index is a multiple of nn
+        #     if i % nn == 0:
+        #         #print("elaborate RAG triples")
+        # 
+        #         #df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=20)
+        #         #df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0,  max_num_passages=20, min_threshold=0.7)
+        #         #df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90,  max_num_passages=20, min_threshold=0.7)
+        #         df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, min_threshold=0.7)
+        # 
+        #         passages = []
+        # 
+        #         df_retrieved = df_retrievedtopk.copy()
+        #         if not df_retrieved.empty:
+        #             labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
+        #             labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
+        # 
+        #             if not labelTriples:
+        #                 labelTriples =labelTriplesAPP
+        #             else:
+        #                 labelTriples = labelTriples + ". " + labelTriplesAPP
+        #
+        # if passages:
+        #     df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, min_threshold=0.7)
+        #
+        #     df_retrieved = df_retrievedtopk.copy()
+        #     if not df_retrieved.empty:
+        #         labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
+        #         labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
+        #         if not labelTriples:
+        #             labelTriples = labelTriplesAPP
+        #         else:
+        #             labelTriples = labelTriples + ". " + labelTriplesAPP
+        #
+        # if labelTriples:
+        #     labelTriples.strip().replace("..",".").strip()
+
+
+    else:  # NO RAG on triples 
         labelTriples = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST)
 
 
@@ -1483,18 +1563,18 @@ def getLinearTextualContextFromTriples(word,labelTriplesLIST, text_splitter, arg
         try:
 
             contextText = ""
-            # if args.service_provider == "gptjrc":
-            #     contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
-            #                       temperature=args.temperature, delimiter=myDelimiter,
-            #                       InContextExamples=[],
-            #                       handler=api_call_gptjrc,
-            #                       verbose=True, args=args)
-            # elif args.service_provider == "HFonPremises":
-            #     contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
-            #                              temperature=args.temperature, delimiter=myDelimiter,
-            #                              InContextExamples=[],
-            #                              handler=api_call_HFonPremises,
-            #                              verbose=True, args=args)
+            if args.service_provider == "gptjrc":
+                contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
+                                  temperature=args.temperature, delimiter=myDelimiter,
+                                  InContextExamples=[],
+                                  handler=api_call_gptjrc,
+                                  verbose=True, args=args)
+            elif args.service_provider == "HFonPremises":
+                contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
+                                         temperature=args.temperature, delimiter=myDelimiter,
+                                         InContextExamples=[],
+                                         handler=api_call_HFonPremises,
+                                         verbose=True, args=args)
 
 
 
@@ -1519,7 +1599,6 @@ def getLinearTextualContextFromTriples(word,labelTriplesLIST, text_splitter, arg
     return contextText, map_query_input_output
 
 
-
 #@mem.cache
 def virtuoso_api_call(word, text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=None, iALLURIScontextFromNCBO=None,UseBioportalForLinking=True,questionText=""):