init

app.py

@@ -6,16 +6,17 @@ import json
 import os
 import time
 import mp_box
-from mp_utils import get_pixel_cordinate_list,extract_landmark,get_pixel_cordinate,get_pixel_xyz
+from mp_estimate  import ratios_cordinates,estimate_horizontal,estimate_vertical,mean_std_label,normalized_to_pixel,get_feature_angles_cordinate,create_detail_labels,get_feature_ratios_cordinate
+from mp_utils import get_pixel_cordinate_list,extract_landmark,get_pixel_cordinate,get_pixel_xyz,get_normalized_landmarks
 from glibvision.draw_utils import points_to_box,box_to_xy,plus_point
 
 
 from glibvision.cv2_utils import plot_points,create_color_image,pil_to_bgr_image,set_plot_text,copy_image
-
+from glibvision.numpy_utils import rotate_point_euler,load_data
 from gradio_utils import save_image,save_buffer,clear_old_files ,read_file
 
 import cv2
-from  cv2_pose_estimate import estimate_head_pose,draw_head_pose
+from  cv2_pose_estimate import draw_head_pose
 
 import numpy as np
 from numpy.typing import NDArray
@@ -25,86 +26,21 @@ from numpy.typing import NDArray
     iris_mask_blur - final iris edge blur
 '''
 
-set_plot_text(False,0.5,(200,200,200))
-
-depath_ratio = 1.0
-
-model_cordinates = [ (0.0, 0.0, 0.0),  # Nose tip
-    (0.0, 344.0, -40.0 * depath_ratio),  # Chin
-    #(0.0, -160.0, -50.0),#center of eye
-    #INNER
-    (-110.0, -215.0, -60.0 * depath_ratio),  #inner Left eye left corner
-    (110.0, -215.0, -60.0 * depath_ratio),  #inner Right eye right corne
-    
-    (-300.0, -250.0, -90.0 * depath_ratio),  # Left eye left corner
-    (300.0, -250.0, -90.0 * depath_ratio),  # Right eye right corne
-
-    (-125.0, 180.0, -70.0 * depath_ratio),  # Left Mouth corner
-    (125.0, 180.0, -70.0 * depath_ratio) ] # Right mouth corner
-
-def fit_cordinates(cordinates,center_x=512,center_y=512,base_distance = 344):
-    ratio = base_distance/(cordinates[1][1])
-    fitted_cordinates = []
-    
-    for cordinate in model_cordinates:
-        fitted_cordinate = [
-            cordinate[0]*ratio+center_x,
-            cordinate[1]*ratio+center_y,
-            cordinate[2]*ratio
-        ]
-        fitted_cordinates.append(fitted_cordinate)
-    return fitted_cordinates
-
-
-def plot_model(cv2_image=None,center_x=512,center_y=512,base_distance = 344):
-    if cv2_image is None:
-        #TODO add arg 
-        cv2_image=create_color_image(np.zeros((1024, 1024,3),dtype=np.uint8))
-    fitted_cordinates = fit_cordinates(model_cordinates,center_x,center_y,base_distance)
-    ratio = base_distance/model_cordinates[1][1]
-
-    def adjust_cordinate(point):
-        
-        
-        return point
-    
-    plot_points(cv2_image,[adjust_cordinate(fitted_cordinates[0])],False,6,(0,0,255),3,(255,0,0))
-    plot_points(cv2_image,[adjust_cordinate((fitted_cordinates[1]))],False,6,(0,0,255),3,(255,0,0))
-
-    plot_points(cv2_image,[adjust_cordinate((fitted_cordinates[2])),adjust_cordinate((fitted_cordinates[4]))],False,6,(0,0,255),3,(255,0,0))
-    plot_points(cv2_image,[adjust_cordinate((fitted_cordinates[3])),adjust_cordinate((fitted_cordinates[5]))],False,6,(0,0,255),3,(255,0,0))
-    plot_points(cv2_image,[adjust_cordinate((fitted_cordinates[6])),adjust_cordinate((fitted_cordinates[7]))],False,6,(0,0,255),3,(255,0,0))
-
-    return cv2_image
-     
-
-def set_model_cordinates(cordinates):
-    global model_cordinates
-    model_cordinates = cordinates
-
 def process_images(image,base_image,
-                   camera_fov,double_check_offset_center,
-                   draw_base_model,fit_base_model,
-                   first_pnp,second_refine,final_iterative,debug_process,draw_mediapipe_mesh,draw_mediapipe_result,z_multiply=0.8,
+                   double_check_offset_center,center_index,
+                   draw_mediapipe_mesh,z_multiply=0.8,draw_mediapipe_angle=False,draw_hozizontal_line=False,draw_vertical_line=False,draw_faceratio_line=False,
         progress=gr.Progress(track_tqdm=True)):
     clear_old_files()
-    
+    """
     image_indices = [4,199,#6,#center of eye
                         133,362,#inner eye
                         33,263, #outer eye
                         61,291]#mouth
-    
-
-    chin = 344
-    global model_cordinates
-    
-    """ normalize ?
-    model_cordinates =[
-        [pt[0]/chin,pt[1]/chin,pt[2]/chin] for pt in model_cordinates
-    ]
     """
     
 
+ 
+
     def landmarks_to_model_corsinates(face_landmarks,indices,w,h):
         cordinates = []
         z_depth = w if w<h else h
@@ -123,24 +59,6 @@ def process_images(image,base_image,
     size = cv2_image.shape
     center: tuple[float, float] = (size[1] / 2, size[0] / 2)
 
-    if base_image is not None:#additiona base image
-        
-        base_image_indices = [
-            6,197,195,5,4,#nose center
-            122,196,  3, 51, 45,
-            351,419,248,281,275,
-
-            122,245,244,243,133, #eyes
-            351,465,464,463,362 #eyes
-        ]
-        # TODO check same?
-        cv2_base_image = pil_to_bgr_image(base_image)
-        mp_image,face_landmarker_result = extract_landmark(cv2_base_image,"face_landmarker.task",0,0,True)
-        h,w = cv2_base_image.shape[:2]
-        
-        image_indices = base_image_indices
-        set_model_cordinates(landmarks_to_model_corsinates(face_landmarker_result.face_landmarks,image_indices,w,h))
-    print(image_indices)
 
     import math
     def calculate_distance(xy, xy2):
@@ -151,70 +69,101 @@ def process_images(image,base_image,
     h,w = im.shape[:2]
 
     first_landmarker_result = None
+    def get_first_landmarker_result():
+        if first_landmarker_result:
+            return first_landmarker_result
+        else:
+            return face_landmarker_result
+
+    first_translation_vector = None
     if double_check_offset_center:
-        root_cordinate = get_pixel_cordinate(face_landmarker_result.face_landmarks,image_indices[0],w,h)#nose tip
+        root_cordinate = get_pixel_cordinate(face_landmarker_result.face_landmarks,center_index,w,h)#nose tip
         diff_center_x = center[0] - root_cordinate[0]
         diff_center_y = center[1] - root_cordinate[1]
         base = np.zeros_like(cv2_image)
         copy_image(base,cv2_image,diff_center_x,diff_center_y)
+        #cv2.imwrite("center.jpg",base)
         first_landmarker_result = face_landmarker_result
         mp_image,face_landmarker_result = extract_landmark(base,"face_landmarker.task",0,0,True)
         im = mp_image.numpy_view()
+        transformation_matrix=first_landmarker_result.facial_transformation_matrixes[0]
+        rotation_matrix, first_translation_vector = transformation_matrix[:3, :3],transformation_matrix[:3, 3]
     else:
         diff_center_x=0
         diff_center_y=0
         #return base,"",""
 
-    cordinates = get_pixel_cordinate_list(face_landmarker_result.face_landmarks,image_indices,w,h)
+    #cordinates = get_pixel_cordinate_list(face_landmarker_result.face_landmarks,image_indices,w,h)
     
 
 
+
+
     if draw_mediapipe_mesh:
-        image = mp_box.draw_landmarks_on_image(face_landmarker_result,image)
-        cv2_image = pil_to_bgr_image(image)
-
-    chin_distance = calculate_distance(cordinates[0],cordinates[1])
-    #trying detect pnp from same pose,but seeems not working
-    #fitted_cordinates = fit_cordinates(model_cordinates,cordinates[0][0],cordinates[0][1],chin_distance)
-    if fit_base_model:
-        #not get good result
-        #model_points: NDArray = np.array(fitted_cordinates, dtype="double")
-        model_points: NDArray = np.array(model_cordinates, dtype="double")
-    else:
-        model_points: NDArray = np.array(model_cordinates, dtype="double")
-    
+        result = first_landmarker_result
+        if result == None:
+            result = face_landmarker_result
+        image = mp_box.draw_landmarks_on_image(result,image)
+        cv2_image = pil_to_bgr_image(image)#here must be bug,but somehow working
+
     
 
+
+
+
+    # draw lines
+
+    #x_ratios = []
+    z_angles,y_ratios,h_cordinates,_ = estimate_horizontal(get_first_landmarker_result().face_landmarks)
    
+    if draw_hozizontal_line:
+        for cordinates in h_cordinates:
+            print(cordinates)
+            points = normalized_to_pixel(cordinates,w,h)
+            print(points)
+            plot_points(cv2_image,points[:2],False,5,(255,0,0),3)#last one is middle point on horizontal
+        
+            
 
+    _,x_ratios,v_cordinates,_ = estimate_vertical(get_first_landmarker_result().face_landmarks)
+    if draw_vertical_line:
+        for cordinates in v_cordinates:
+            plot_points(cv2_image,normalized_to_pixel(cordinates,w,h),False,5,(0,0,255),3,(255,0,0))#second one is middle point on vertical
 
+    #these are for training feature
+    key_cordinates,angles = get_feature_angles_cordinate(get_first_landmarker_result().face_landmarks)
+    for cordinates in key_cordinates:
+            pass
+            #plot_points(cv2_image,normalized_to_pixel(cordinates,w,h),False,5,(0,0,255),3,(255,0,0))
+    key_cordinates,angles = get_feature_ratios_cordinate(get_first_landmarker_result().face_landmarks)
+    for cordinates in key_cordinates:
+            pass
+            #plot_points(cv2_image,normalized_to_pixel(cordinates,w,h),False,5,(0,0,255),3,(255,0,0))
+               
 
-    focal_length: float = calculate_distance(cordinates[0],cordinates[1])
-    focal_length = focal_length*camera_fov
+    z_angle_text = mean_std_label(z_angles,True)
+    y_ratio_text = mean_std_label(y_ratios)
+    x_ratio_text = mean_std_label(x_ratios)
 
-    
-    
-    
-    #image_size = size[0] #TODO
-    #f = (image_size / 2) / np.tan(np.deg2rad(camera_fov / 2))
-    #focal_length = f
-    #print(f"fov ={camera_fov} size = {image_size} focal_length = {focal_length}")
+    z_angle_detail = create_detail_labels(z_angles,True)
+    y_ratio_detail = create_detail_labels(y_ratios)
+    x_ratio_detail = f"forehead-chin = {np.mean(x_ratios)}"
 
-    
+
+    focal_length: float = calculate_distance(cordinates[0],cordinates[1])
+    focal_length = focal_length*1
     
     camera_matrix: NDArray = np.array([
         [focal_length, 0, center[0]],
-        [0, focal_length, center[1]],
+        [0, -focal_length, center[1]],
         [0, 0, 1]
     ], dtype="double")
     dist_coeffs: NDArray = np.zeros((4, 1))
 
     # offset center usually improve result
 
-
     image_points: NDArray = np.array(cordinates, dtype="double")
 
-
     from scipy.spatial.transform import Rotation as R
     def print_euler(rotation_vector,label=""):
         order = "yxz"
@@ -223,13 +172,11 @@ def process_images(image,base_image,
         r = R.from_matrix(rotation_matrix)
         euler_angles = r.as_euler(order, degrees=True)
         label = f"{label} Euler Angles {order} (degrees): {euler_angles}"
-        
         return label
 
     rotation_vector = None
     translation_vector = None
     im_with_pose = cv2_image
-    result_label = None
     mediapipe_text = None
 
     def face_landmarker_result_to_angle_label(face_landmarker_result,order="yxz"):
@@ -239,85 +186,203 @@ def process_images(image,base_image,
             
             rotation_matrix, translation_vector = transformation_matrix[:3, :3],transformation_matrix[:3, 3]
             #TODO change base-size
-            scaled_translation_vector =(translation_vector[0]*1024,translation_vector[1]*1024,translation_vector[2]*1024)
+            vector_multiply=10
+            scaled_translation_vector =(translation_vector[0]*vector_multiply,translation_vector[1]*vector_multiply,translation_vector[2]*vector_multiply)
             #scaled_translation_vector = (-512,-512,-1024)
-            if draw_mediapipe_result:
-                im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_matrix, scaled_translation_vector, camera_matrix, dist_coeffs,32,-diff_center_x,-diff_center_y)
+            #im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_matrix, scaled_translation_vector, camera_matrix, dist_coeffs,32,-diff_center_x,-diff_center_y)
             #print("mediapipe",scaled_translation_vector)
             #mediapipe_label = print_euler(rotation_vector,"MediaPipe")
             r = R.from_matrix(rotation_matrix)
             euler_angles = r.as_euler(order, degrees=True)
-            label = f"Media pipe Euler Angles {order} (degrees): {euler_angles}"
-            return label
+            #label = f"Media pipe {order}-Euler Angles [x,y,z] (degrees): [{euler_angles[1]:.2f},{euler_angles[0]:.2f},{euler_angles[2]:.2f}]"
+            label = f"[x:{euler_angles[1]:.2f},y:{-euler_angles[0]:.2f},z:{-euler_angles[2]:.2f}]"
+            
+            return label,rotation_matrix,scaled_translation_vector
         
     if first_landmarker_result != None:
-        mediapipe_first_text = face_landmarker_result_to_angle_label(first_landmarker_result)
+        mediapipe_first_text,_,_ = face_landmarker_result_to_angle_label(first_landmarker_result)
     else:
         mediapipe_first_text = ""
         
-    mediapipe_second_text = face_landmarker_result_to_angle_label(face_landmarker_result)
-
-    if first_pnp!="None":
-        if first_pnp == "EPNP":
-            flags = cv2.SOLVEPNP_EPNP
-        elif first_pnp == "ITERATIVE":
-            flags = cv2.SOLVEPNP_ITERATIVE
-        elif first_pnp == "IPPE":
-            flags = cv2.SOLVEPNP_IPPE
-        else:
-            flags = cv2.SOLVEPNP_SQPNP
-        if first_pnp == "Mediapipe":
-            rotation_vector, _ = cv2.Rodrigues(rotation_matrix)
-            translation_vector =  scaled_translation_vector
-        else:
-            translation_vector = None
-            #translation_vector = np.array([cordinates[0][0],cordinates[0][1],focal_length],dtype="double")
-            #translation_vector = scaled_translation_vector
-            #print("initial",translation_vector,)
-            rotation_vector, translation_vector = estimate_head_pose(cv2_image, model_points,image_points, camera_matrix, dist_coeffs,flags,None,translation_vector)
-            #print(translation_vector)
-            im_with_pose = cv2_image
-            result_label = print_euler(rotation_vector,first_pnp)
-            print("firstpnp",translation_vector)
-            if debug_process:
-                im_with_pose = draw_head_pose(cv2_image, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs,128,-diff_center_x,-diff_center_y)
-
-    if first_pnp!="None" and second_refine!="None":
-        criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 1000, 1e-8)  # 反復終了条件
-        if second_refine == "LM":
-            rotation_vector, translation_vector = cv2.solvePnPRefineLM(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector, criteria=criteria)
-        else:
-            rotation_vector, translation_vector = cv2.solvePnPRefineVVS(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector, criteria=criteria)
-        if debug_process:
-            im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs,128+64,-diff_center_x,-diff_center_y)
-        result_label = print_euler(rotation_vector,second_refine)
-        #print("refine",translation_vector)
-
-    if final_iterative:
-        (success, rotation_vector, translation_vector) = cv2.solvePnP(
-            model_points, image_points, camera_matrix, dist_coeffs,rotation_vector ,translation_vector,flags=cv2.SOLVEPNP_ITERATIVE)
-        if success:
-             result_label = print_euler(rotation_vector,"SOLVEPNP_ITERATIVE")
-        else:
-            
-            raise gr.Warning("final_iterative faild")
-    #draw final one
-    if rotation_vector is not None:
-        im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs,255,-diff_center_x,-diff_center_y)
+    mediapipe_second_text,rotation_matrix,scaled_translation_vector = face_landmarker_result_to_angle_label(face_landmarker_result)
+
+    rotation_vector, _ = cv2.Rodrigues(rotation_matrix)
+    translation_vector =  scaled_translation_vector
+
+    #if first_translation_vector.all():
+    #    translation_vector = first_translation_vector
+    #im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs,255,-diff_center_x,-diff_center_y)
         
         # mediapipe metrix
         #print("opencv",translation_vector)
+
+   
+    if draw_mediapipe_angle:
+        root_cordinate = get_pixel_xyz(get_first_landmarker_result().face_landmarks,4,w,h)
     
+        r = R.from_matrix(rotation_matrix)
+        euler_angles = r.as_euler("yxz", degrees=False)
+        print(r.as_euler("yxz", degrees=True))
+        draw_cordinate1=rotate_point_euler((0,0,-100),[-euler_angles[1],euler_angles[0],euler_angles[2]],"yzx")
+        draw_cordinate2=rotate_point_euler((0,0,-200),[-euler_angles[1],euler_angles[0],euler_angles[2]],"yzx")
+        
+        plot_points(im_with_pose,[root_cordinate[:2]+draw_cordinate1[:2],root_cordinate[:2]+draw_cordinate2[:2],root_cordinate[:2]],False,5,(0,128,0),3,(0,255,0))
+
+    #analyze face ratios
+    landmarks = get_normalized_landmarks(get_first_landmarker_result().face_landmarks)
+    face_ratio_infos = []
+   
+
+    print("landmark",[landmarks[37],landmarks[267]])
+    print("numpy",np.array([landmarks[37],landmarks[267]]))
+    print("mean",np.mean(np.array([landmarks[37],landmarks[267]]),axis=0))
+    v_cordinates=[
+    ["philtrum",landmarks[175],landmarks[13],np.mean((landmarks[164],landmarks[2]),axis=0).tolist()],
+    ["straight",landmarks[175],landmarks[94],landmarks[9]],
+    ["face",landmarks[175],landmarks[9],landmarks[127],landmarks[356]],
+    ["r-eyes",landmarks[33],landmarks[190],landmarks[414]],
+    ["r-contour",landmarks[127],landmarks[33],landmarks[190]],
+    ["l-eyes",landmarks[263],landmarks[414],landmarks[190]],
+    ["l-contour",landmarks[356],landmarks[263],landmarks[414]],
+    ["lips",landmarks[17],landmarks[13],np.mean((landmarks[37],landmarks[267]),axis=0).tolist()],
+    ["mouth-eye",landmarks[61],landmarks[291],landmarks[133],landmarks[362]],
+    ]
     
-    if draw_base_model:
-        if fit_base_model:
-            im_with_pose=plot_model(im_with_pose,cordinates[0][0],cordinates[0][1],chin_distance)
-        else:
-             im_with_pose=plot_model(im_with_pose)
+    for cordinates in v_cordinates:
+            ratio=ratios_cordinates(cordinates[1:])
+            if draw_faceratio_line:
+                plot_points(cv2_image,normalized_to_pixel(cordinates[1:],w,h),False,5,(0,255,255),3,(255,255,0))
+            label = f"{cordinates[0]}:{ratio:.2f}"
+            face_ratio_infos.append(label)
+    face_ratio_info=",".join(face_ratio_infos)
+    return cv2.cvtColor(im_with_pose,cv2.COLOR_BGR2RGB),mediapipe_first_text,mediapipe_second_text,z_angle_text,y_ratio_text,x_ratio_text,z_angle_detail,y_ratio_detail,x_ratio_detail,face_ratio_info
+
+
+
+def find_nearest_weighted_euclidean_2d(target_angles_full, all_angles_full, weights):
+    target_angles = target_angles_full[:5]  # 最初の3つの角度を使用
+    all_angles = all_angles_full[:, :5]  # 最初の3列を使用
+
+    weighted_diff = (all_angles - target_angles) * weights
+    distances = np.linalg.norm(weighted_diff, axis=1)
+    nearest_index = np.argmin(distances)
+    return nearest_index, all_angles_full[nearest_index]
+
+from mp_estimate import estimate_horizontal_points ,estimate_vertical_points,estimate_rotations_v2
+def find_angles(image):
+    if image is None:
+         raise gr.Error("need image")
+    cv2_image = pil_to_bgr_image(image)
+    size = cv2_image.shape
+    mp_image,face_landmarker_result = extract_landmark(cv2_image,"face_landmarker.task",0,0,True)
+    
+    
+    features_text = estimate_rotations_v2(face_landmarker_result)
+    features_value_origin = [float(value) for value in features_text.split(",")]
+    features_value = features_value_origin.copy()
+    #print(features_value)
+    #weights = np.array([0.2, 0.2,0.3,0.3])
+
+    #index,matched = find_nearest_weighted_euclidean_2d(target_angles,all_angles,weights)
+    #index,matched = find_nearest_euclidean_2d(target_angles,all_angles)
+    #formatted_arr = [np.format_float_positional(x) for x in matched]
+    #print(formatted_arr)
+    x_ratios = 11 #magic vertical ratios
+    
+    #short
+    features_values = [
+        [np.add(features_value[-x_ratios:],features_value[0:1])],
+        [features_value[:-x_ratios]],
+        [np.hstack([features_value[ 3:5],features_value[ 6:-x_ratios]])]
+        #[features_value[:-x_ratios]]
+    ]
+    import joblib
+    def estimate(model_path,scaler_path,features_values):
+        scalers = joblib.load("models/"+scaler_path)
+        if not isinstance(scalers,list):
+            scalers=(scalers,scalers,scalers)
+        for i,scaler  in enumerate(scalers):
+            print(i,scaler)
+            features_values[i] = scaler.transform(features_values[i].copy())
+
+
+        result_preds=[]
+        models = joblib.load("models/"+model_path)
+        for i,model in enumerate(models):
+            y_pred = model.predict(features_values[i])
+            result_preds.append(y_pred.round(2))
+        return result_preds
+    def estimate2(model_key,features_values):
+        model_path=f"models/{model_key}.joblib"
+        scaler_path=f"models/{model_key}_scaler.joblib"
+        polynomial_path=f"models/{model_key}_polynomial_features.joblib"
+        selectkbest_path=f"models/{model_key}_selectkbest.joblib"
+        model = joblib.load(model_path)
+        scaler = joblib.load(scaler_path)
+        polynomial = joblib.load(polynomial_path)
+        selectkbest = joblib.load(selectkbest_path)
+        result_preds=[]
+        for i in range(3):
+            x = polynomial[i].transform(features_values[i].copy())
+            x = selectkbest[i].transform(x)
+            x = scaler[i].transform(x)
+            y_pred = model[i].predict(x)
+            result_preds.append(y_pred.round(2))
+        return result_preds
+
+
+    #short_result = estimate('linear-svr-xyz_5.joblib','linear-svr-xyz_5_scaler.joblib',features_values)
+
+    features_value = features_value_origin.copy()
+    features_values = [
+        [features_value],[features_value],[features_value]
+    ]
+    #short_result = estimate('lgbm-optimizer_15.joblib','lgbm-optimizer_15_scaler.joblib',features_values.copy())
+    short_result = estimate2('hyper-hgr-random15',features_values.copy())
 
-    return cv2.cvtColor(im_with_pose,cv2.COLOR_BGR2RGB),result_label,mediapipe_first_text,mediapipe_second_text
+    
+   
+    
+    #middle_result = estimate('lgbm-xyz_90-rand47.joblib','lgbm-xyz_90-rand47_scaler.joblib',features_values.copy())
+    middle_result = estimate2('hyper-hgr-random45',features_values.copy())
+    
+    long_result = estimate2('hyper-hgr-random90',features_values.copy())
+    
 
+    e1_key="lgbm-optimizer_15dart_random"
+    short_result2a = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy())
+    e1_key="lgbm-optimizer_15_random"
+    short_result2 = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy())
 
+    e1_key="lgbm-optimizer_45_random"
+    
+    middle_result2 = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy())
+    e1_key="lgbm-optimizer_90_random"
+    long_result2 = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy())
+    def flatten_for(lst):
+        return [round(item, 3) for sublist in lst for item in sublist]
+        
+    def average(values):
+        flat_values=[]
+        for value in values:
+             flat_values += [flatten_for(value)]
+        print(np.mean(flat_values,axis=0))
+
+    import average
+    data={
+         "hgbr-15":flatten_for(short_result),
+         "hgbr-45":flatten_for(middle_result),
+         "hgbr-90":flatten_for(long_result),
+         "lgbm-15dart":flatten_for(short_result2a),
+         "lgbm-15":flatten_for(short_result2),
+         "lgbm-45":flatten_for(middle_result2),
+         "lgbm-90":flatten_for(long_result2),
+    }
+    #print(data)
+    average_data=average.analyze_3d_data(data.values())
+    print(average_data)
+    #average((short_result,middle_result,long_result,short_result2a,short_result2,middle_result2,long_result2))
+    return average_data['trimmed_mean'],flatten_for(short_result),flatten_for(middle_result),flatten_for(long_result),flatten_for(short_result2a),flatten_for(short_result2),flatten_for(middle_result2),flatten_for(long_result2)
 
 css="""
 #col-left {
@@ -360,7 +425,7 @@ with gr.Blocks(css=css, elem_id="demo-container") as demo:
                     
                     with gr.Row(elem_id="prompt-container",  equal_height=False):
                         with gr.Row():
-                            btn = gr.Button("Pose Estimate", elem_id="run_button",variant="primary")
+                            btn = gr.Button("Head-Pose Estimate", elem_id="run_button",variant="primary")
                     
                     
                         
@@ -369,45 +434,67 @@ with gr.Blocks(css=css, elem_id="demo-container") as demo:
                         base_image = gr.Image(sources=['upload','clipboard'],image_mode='RGB',elem_id="image_upload", type="pil", label="Image",visible=False)
                        
                         with gr.Row( equal_height=True):
-                            camera_fov = gr.Slider(info="not effect mediapipe,nose-chin x multiply",
-                            label="Multiply value",
-                            minimum=0.1,
-                            maximum=2.0,
-                            step=0.01,
-                            value=1.2)
-                            double_check_offset_center = gr.Checkbox(label="offset center point",value=True,info="move center and detect again(usually more accurate)") 
-                            z_multiply = gr.Slider(info="nose depth",
-                            label="Z-Multiply",
+                            
+                            double_check = gr.Checkbox(label="Double Check",value=True,info="move center and detect again(usually more accurate).recommend choose 195") 
+                            center_index = gr.Slider(info="center-index",
+                            label="Center-index",
+                            minimum=0,
+                            maximum=467,
+                            step=1,
+                            value=195)
+                            z_multiply = gr.Slider(info="nose height",
+                            label="Depth-Multiply",
                             minimum=0.1,
                             maximum=1.5,
                             step=0.01,
                             value=0.8)
-                        with gr.Row( equal_height=True):
-                            draw_base_model = gr.Checkbox(label="draw base model",value=False,info="draw base model") 
-                            fit_base_model = gr.Checkbox(label="fit base model",value=False,info="This is just for visual,not use as model")
                         
-                        first_pnp =gr.Radio(label="PnP",choices=["None","EPNP","SQPNP","IPPE","ITERATIVE","Mediapipe"],value="EPNP")
-                        second_refine =gr.Radio(label="PnP refine",choices=["None","LM","VVS"],value="LM")
                         with gr.Row( equal_height=True):      
-                            final_iterative = gr.Checkbox(label="PnP final iterative",value=False,info="sometime good")
-                            debug_process = gr.Checkbox(label="Debug Process",value=False)
-                            draw_mediapipe_mesh = gr.Checkbox(label="Draw mediapipe mesh",value=False)
-                            draw_mediapipe_result = gr.Checkbox(label="Draw mediapipe result",value=False)
-                        plot_button = gr.Button("Plot Model", elem_id="run_button")
+                            draw_mediapipe_mesh = gr.Checkbox(label="Draw mediapipe mesh",value=True)
+                            draw_mediapipe_angle = gr.Checkbox(label="Draw mediapipe angle(green)",value=True)
+                        with gr.Row( equal_height=True):
+                            draw_hozizontal_line = gr.Checkbox(label="Draw horizontal line(red)",value=True)
+                            draw_vertical_line = gr.Checkbox(label="Draw vertical line(blue)",value=True)
+                            draw_faceratio_line = gr.Checkbox(label="Draw Face-Ratio line(blue)",value=False)
                                          
                 with gr.Column():
                     result_image = gr.Image(height=760,label="Result", elem_id="output-animation",image_mode='RGB')
-                    result_text = gr.Textbox(label="cv2 result")
-                    mediapipe_first_text = gr.Textbox(label="first mediapipe result")
-                    mediapipe_last_text = gr.Textbox(label="2nd or last mediapipe result")
+                    with gr.Row( equal_height=True):
+                        mediapipe_last_text = gr.Textbox(label="2nd or last mediapipe result(yzx-eulder[x,y,z])")
+                        mediapipe_first_text = gr.Textbox(label="first mediapipe result(yzx-eulder[x,y,z])")
+                    
+                    with gr.Row( equal_height=True):
+                        z_angle_text = gr.Textbox(label="Z angle by horizontal-line",info="start with 0,exactly Z-Angle")
+                        y_ratio_text = gr.Textbox(label="Y Left-Right length ratio",info="start 0.49-0.51")
+                        x_ratio_text = gr.Textbox(label="X Up-down length ratio",info="start near 0.49,look at nose-hole-shape")
+                    with gr.Accordion(label="Angle Ratio Details", open=False):
+                        with gr.Row( equal_height=True):
+                            z_angle_detail_text = gr.TextArea(label="Z-angle detail")
+                            y_ratio_detail = gr.TextArea(label="Y-ratio detail")
+                            x_ratio_detail = gr.TextArea(label="X-ratio detail",value="")
+                    with gr.Row( equal_height=True):
+                        face_ratio_info = gr.Text(label="Face Ratio",info="Average philtrum:1.82(std 0.13),straight:0.82(std 0.04),face:0.91(std 0.02),r-eyes:0.86(std 0.03),r-contour:0.77(std 0.05),l-eyes:0.86(std 0.03),l-contour:0.75(std 0.05),lips:1.43(std 0.16),mouth-eye:1.21(std 0.07)")
+                    gr.HTML("<h5>For Rotation sometime differenct to mediapipe's result(Especially X usually minus 4-7)</h5>")
+                    bt_test = gr.Button("Model-Estimate")
+                    gr.HTML("<p>YXZ-Euler [x,y,z] hgbr is stable,lgbm is accurate(dart is more).trimmed works well on small angles</p>")  
+                    with gr.Row( equal_height=True):  
+                            average_result = gr.Text(label="trimmed-mean")
+                            short_result = gr.Text(label="hgbr-15")
+                            middle_result = gr.Text(label="hgbr-45")
+                            long_result = gr.Text(label="hgbr-90")
+                    with gr.Row( equal_height=True):  
+                            short_result2a = gr.Text(label="lgbm-15dart")
+                            short_result2 = gr.Text(label="lgbm-15")
+                            middle_result2 = gr.Text(label="lgbm-45")
+                            long_result2 = gr.Text(label="lgbm-90")
+                            #,
+                            bt_test.click(fn=find_angles,inputs=image,outputs=[average_result,short_result,middle_result,long_result,short_result2a,short_result2,middle_result2,long_result2])
 
     btn.click(fn=process_images, inputs=[image,base_image,
-                                         camera_fov,double_check_offset_center,
-                                         draw_base_model,fit_base_model,
-                                         first_pnp,second_refine,final_iterative,debug_process,draw_mediapipe_mesh,draw_mediapipe_result
-                                         ],outputs=[result_image,result_text,mediapipe_first_text,mediapipe_last_text] ,api_name='infer')
-    plot_button.click(fn=plot_model,inputs=[],outputs=[result_image])
-    
+                                         double_check,center_index,
+                                         draw_mediapipe_mesh,z_multiply,draw_mediapipe_angle,draw_hozizontal_line,draw_vertical_line,draw_faceratio_line,
+                                         ],outputs=[result_image,mediapipe_first_text,mediapipe_last_text,z_angle_text,y_ratio_text,x_ratio_text,z_angle_detail_text,y_ratio_detail,x_ratio_detail,face_ratio_info] ,api_name='infer')
+   
     example_images = [
                      ["examples/02316230.jpg"],
                     ["examples/00003245_00.jpg"],

average.py

@@ -0,0 +1,102 @@
+import numpy as np
+
+def analyze_3d_data(data):
+    """
+    3Dデータポイントのリストを受け取り、トリム平均とSTD法による外れ値を返す
+
+    Args:
+        data: [(x1, y1, z1), (x2, y2, z2), ...] 形式のリスト
+
+    Returns:
+        dict: 
+          {
+            'trimmed_mean': (x_mean, y_mean, z_mean),  # トリム平均
+            'std_outliers': {
+                'x': [x_outlier1, x_outlier2, ...],  # x座標の外れ値
+                'y': [y_outlier1, y_outlier2, ...],  # y座標の外れ値
+                'z': [z_outlier1, z_outlier2, ...]   # z座標の外れ値
+            },
+            'trimmed_data': {
+                'x': [x1, x2, ...], # トリム平均に使用したx座標データ
+                'y': [y1, y2, ...], # トリム平均に使用したy座標データ
+                'z': [z1, z2, ...]  # トリム平均に使用したz座標データ
+            }
+          }
+    """
+
+    # データを x, y, z 座標ごとに整理
+    x_coords = [point[0] for point in data]
+    y_coords = [point[1] for point in data]
+    z_coords = [point[2] for point in data]
+
+    def trimmed_data_and_mean(coords):
+        """
+        最小値と最大値を除外したデータとトリム平均を返す
+        """
+        coords_sorted = sorted(coords)
+        trimmed_coords = coords_sorted[1:-1]  # 最小値と最大値を除外
+        return trimmed_coords, np.mean(trimmed_coords)
+
+    # 各座標のトリム平均を計算
+    x_trimmed, x_trimmed_mean = trimmed_data_and_mean(x_coords)
+    y_trimmed, y_trimmed_mean = trimmed_data_and_mean(y_coords)
+    z_trimmed, z_trimmed_mean = trimmed_data_and_mean(z_coords)
+
+    def detect_outliers_std(data, multiplier=2):
+        """
+        標準偏差に基づく外れ値検出 (multiplier=2)
+        """
+        mean = np.mean(data)
+        std = np.std(data)
+        lower_bound = mean - multiplier * std
+        upper_bound = mean + multiplier * std
+        outliers = [x for x in data if x < lower_bound or x > upper_bound]
+        return outliers
+
+    # トリム後のデータに対して STD 法を適用
+    x_outliers_std = detect_outliers_std(x_trimmed, multiplier=2)
+    y_outliers_std = detect_outliers_std(y_trimmed, multiplier=2)
+    z_outliers_std = detect_outliers_std(z_trimmed, multiplier=2)
+
+    return {
+        'trimmed_mean': (round(x_trimmed_mean,2), round(y_trimmed_mean,2), round(z_trimmed_mean,2)),
+        'std_outliers': {
+            'x': x_outliers_std,
+            'y': y_outliers_std,
+            'z': z_outliers_std,
+        },
+        'trimmed_data': {
+            'x': x_trimmed,
+            'y': y_trimmed,
+            'z': z_trimmed
+        }
+    }
+"""
+# 使用例 (元のデータを使用)
+data = {
+    'hgbr-15': [-5.4, 1.56, -2.92],
+    'hgbr-45': [-4.5, 1.2, -1.76],
+    'hgbr-90': [1.58, 1.82, -3.35],
+    'lgbm-15dart': [-6.18, 3.11, -2.46],
+    'lgbm-15': [-5.65, 1.76, -2.59],
+    'lgbm-45': [-7.18, 1.42, -2.71],
+    'lgbm-90': [-3.58, 3.94, -2.5],
+}
+
+# 辞書型のデータをリスト型に変換
+data_list = list(data.values())
+
+# 関数を呼び出し
+result = analyze_3d_data(data_list)
+
+# 結果を表示
+print("トリム平均:", result['trimmed_mean'])
+print("STD法による外れ値 (multiplier=2):")
+print("  x:", result['std_outliers']['x'])
+print("  y:", result['std_outliers']['y'])
+print("  z:", result['std_outliers']['z'])
+print("トリム平均に使用したデータ:")
+print("  x:", result['trimmed_data']['x'])
+print("  y:", result['trimmed_data']['y'])
+print("  z:", result['trimmed_data']['z'])
+"""

cv2_pose_estimate.py

@@ -1,273 +0,0 @@
-import cv2
-import numpy as np
-from numpy.typing import NDArray
-import sys
-from mp_utils import get_pixel_cordinate_list,extract_landmark
-def estimate_head_pose(im: NDArray, model_points: NDArray, image_points,camera_matrix: NDArray, dist_coeffs: NDArray,flags = cv2.SOLVEPNP_ITERATIVE,rotation_vector=None,translation_vector=None) -> tuple[NDArray, NDArray]:
-    """
-    Estimates the head pose from an image.
-
-    Args:
-        image_path: Path to the image file.
-        model_points: 3D model points.
-        camera_matrix: Camera intrinsic matrix.
-        dist_coeffs: Lens distortion coefficients.
-
-    Returns:
-        rotation_vector: Estimated rotation vector.
-        translation_vector: Estimated translation vector.
-    """
-    size = im.shape
-
-    '''
-    image_points: NDArray = np.array([
-        (359, 391),  # Nose tip
-        (399, 561),  # Chin
-        (337, 297),  # Left eye left corner
-        (513, 301),  # Right eye right corne
-        (345, 465),  # Left Mouth corner
-        (453, 469)  # Right mouth corner
-    ], dtype="double")
-'''
-
-    model_points = model_points +500
-    (success, rotation_vector, translation_vector) = cv2.solvePnP(
-        model_points, image_points, camera_matrix, dist_coeffs,flags=flags,
-    )
-    print(model_points)
-    print(image_points)
-    print(camera_matrix)
-
-    if not success:
-        raise RuntimeError("solvePnP failed.")
-
-    return rotation_vector, translation_vector
-
-import cv2
-import numpy as np
-from numpy.typing import NDArray
-
-def draw_head_pose(image: NDArray, image_points: NDArray, rotation_vector: NDArray, translation_vector: NDArray, camera_matrix: NDArray, dist_coeffs: NDArray,color_max=255,offset_x=0,offset_y=0) -> NDArray:
-    """
-    Draws the head pose (XYZ axes) on the image.
-
-    Args:
-        image: Input image.
-        image_points: 2D image points.
-        rotation_vector: Estimated rotation vector.
-        translation_vector: Estimated translation vector.
-        camera_matrix: Camera intrinsic matrix.
-        dist_coeffs: Lens distortion coefficients.
-
-    Returns:
-        Image with head pose drawn.
-    """
-
-    # Define the 3D points for the XYZ axes
-    axis_length = 500.0  # Length of the axes
-    axis_points_3D: NDArray = np.array([
-        [0, 0, 0],      # Origin
-        [axis_length, 0, 0],  # X axis
-        [0, axis_length, 0],  # Y axis
-        [0, 0, axis_length]   # Z axis
-    ], dtype='float32')
-
-    # Project the 3D points to the 2D image plane
-    (axis_points_2D, _) = cv2.projectPoints(
-        axis_points_3D, rotation_vector, translation_vector, camera_matrix, dist_coeffs
-    )
-    axis_points_2D = axis_points_2D.astype(int)
-
-    # Draw the axes on the image
-    origin = tuple(axis_points_2D[0].ravel())
-    cv2.line(image, origin, tuple(axis_points_2D[1].ravel()), (0, 0, color_max), 3)  # X axis (Red)
-    cv2.line(image, origin, tuple(axis_points_2D[2].ravel()), (0, color_max, 0), 3)  # Y axis (Green)
-    cv2.line(image, origin, tuple(axis_points_2D[3].ravel()), (color_max, 0, 0), 3)  # Z axis (Blue)
-
-    for p in image_points:
-        cv2.circle(image, (int(p[0]+offset_x), int(p[1]+offset_y)), 3, (0, 0, 255), -1)
-
-    return image
-
-def main():
-    # 3D model points.
-    '''
-     model_points: NDArray = np.array([
-        (0.0, 0.0, 0.0),  # Nose tip
-        (0.0, 300.0, -65.0),  # Chin
-        (-225.0, -170.0, -135.0),  # Left eye left corner
-        (225.0, -170.0, -135.0),  # Right eye right corne
-        (-150.0, -150.0, -125.0),  # Left Mouth corner
-        (150.0, -150.0, -125.0)  # Right mouth corner
-    ])
-    '''
-    
-    model_points: NDArray = np.array([
-        (0.0, 0.0, 0.0),  # Nose tip
-        (0.0, -344.0, -40.0),  # Chin
-        #(0.0, -160.0, -50.0),#center of eye
-        (-110.0, 215.0, -60.0),  #inner Left eye left corner
-        (110.0, 215.0, -60.0),  #inner Right eye right corne
-        (-300.0, 250.0, -90.0),  # Left eye left corner
-        (300.0, 250.0, -90.0),  # Right eye right corne
-        (-185.0, -180.0, -70.0),  # Left Mouth corner
-        (185.0, -180.0, -70.0)  # Right mouth corner
-    ])
-
-  
-    
-    """
-    
-    model_points: NDArray = np.array([
-        (0.0, 0.0, 0.0),  # Nose tip
-        (0.0, -450.0, 0.0),  # Chin
-        (-110.0, 175.0, -20.0),  #inner Left eye left corner
-        (110.0, 175.0, -20.0),  #inner Right eye right corne
-        (-300.0, 200.0, -40.0),  # Left eye left corner
-        (300.0, 200.0, -40.0),  # Right eye right corne
-        (-176.0, -200.0, -20.0),  # Left Mouth corner
-        (175.0, -200.0, -20.0)  # Right mouth corner
-    ])
-    """
-
-    square_model_points: NDArray = np.array([
-        (-100.0, -100.0, 0),  # Left eye left corner
-        (100.0, -100.0, 0),  # Right eye right corne
-        (-100.0, 100.0, 0),  # Left Mouth corner
-        (100.0, 100.0, 0)  # Right mouth corner
-    ])
-
-
-    # Example image and camera parameters (replace with actual values)
-    image_path = sys.argv[1]
-    mp_image,face_landmarker_result = extract_landmark(image_path)
-    im = mp_image.numpy_view()
-    h,w = im.shape[:2]
-    cordinates = get_pixel_cordinate_list(face_landmarker_result.face_landmarks,[4,199,#6,#center of eye
-                                                                                 33,263,133,362,61,291],w,h)
-    print(cordinates)
-    image_points: NDArray = np.array(cordinates, dtype="double")
-
-    
-
-    import math
-    def calculate_distance(xy, xy2):
-        return math.sqrt((xy2[0] - xy[0])**2 + (xy2[1] - xy[1])**2)
-
-    if im is None:
-        raise FileNotFoundError(f"Could not open or find the image file: {image_path}")
-    size = im.shape
-    focal_length: float = calculate_distance(cordinates[0],cordinates[1])
-    focal_length = focal_length*1.5
-    print("focal length",focal_length)
-    center: tuple[float, float] = (size[1] / 2, size[0] / 2)
-    center = cordinates[0]
-    camera_matrix: NDArray = np.array([
-        [focal_length, 0, center[0]],
-        [0, focal_length, center[1]],
-        [0, 0, 1]
-    ], dtype="double")
-    dist_coeffs: NDArray = np.zeros((4, 1))  # Assuming no lens distortion
-
-    # 2D image points. If you change the image, you need to change vector
-    '''
-        image_points: NDArray = np.array([
-        (321, 571),  # Nose tip
-        (423, 852),  # Chin
-        (201, 406),  # Left eye left corner
-        (529, 363),  # Right eye right corne
-        (336, 705),  # Left Mouth corner
-        (483, 693)  # Right mouth corner
-    ], dtype="double")
-    '''
-    """
-        image_points: NDArray = np.array([
-        #(663, 325),  # Nose tip
-        (655,388),
-        (705, 555),  # Chin
-        (549, 296),  # inner Left eye left corner
-        (651, 291),  # inner Right eye right corne
-        (453, 303),  # Left eye left corner
-        (718, 294),  # Right eye right corne
-        (591, 474),  # Left Mouth corner
-        (715, 472)  # Right mouth corner
-    ], dtype="double")
-    """
-
-
-    square_image_points: NDArray = np.array([
-        (549, 296),  # Nose tip
-        (651, 291),  # Chin
-        (573, 386),  # Left eye left corner
-        (691, 370),  # Right eye right corne
-    ], dtype="double")
-
-    flags_list = [
-        cv2.SOLVEPNP_EPNP#cv2.SOLVEPNP_ITERATIVE#,cv2.SOLVEPNP_SQPNP,cv2.SOLVEPNP_EPNP
-    ]
-    im_with_pose = im.copy()
-    for flags in flags_list:
-        rotation_vector, translation_vector = estimate_head_pose(image_path, model_points,image_points, camera_matrix, dist_coeffs,flags)
-        #print(f"Rotation Vector:\n {rotation_vector}")
-        #print(f"Translation Vector:\n {translation_vector}")
-        #initial
-        #im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs)
-
-    from scipy.spatial.transform import Rotation as R
-    def print_euler(rotation_vector):
-        order = "yxz"
-        rotation_matrix, _ = cv2.Rodrigues(rotation_vector)
-       
-        r = R.from_matrix(rotation_matrix)
-        euler_angles = r.as_euler(order, degrees=True)
-        print(f"Euler Angles {order} (degrees): {euler_angles}")
-
-    print_euler(rotation_vector)
-    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 1000, 1e-8)  # 反復終了条件
-    
-    rotation_vector, translation_vector = cv2.solvePnPRefineLM(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector, criteria=criteria)
-    im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs,128)
-    print_euler(rotation_vector)
-    
-    #rotation_vector[0]=0
-    #rotation_vector[1]=0
-    #rotation_vector[2]=0
-
-    #(success, rotation_vector, translation_vector) = cv2.solvePnP(
-    #    model_points, image_points, camera_matrix, dist_coeffs,rotation_vector ,translation_vector,flags=cv2.SOLVEPNP_ITERATIVE)
-    
-    im_with_pose = draw_head_pose(im_with_pose, image_points, rotation_vector, translation_vector, camera_matrix, dist_coeffs)
-
-    #print_euler(rotation_vector)
-
-    (rotation_matrix, jacobian) = cv2.Rodrigues(rotation_vector)
-    mat = np.hstack((rotation_matrix, translation_vector))
-
-        #yaw,pitch,rollの取り出し
-    (_, _, _, _, _, _, eulerAngles) = cv2.decomposeProjectionMatrix(mat)
-    print(eulerAngles)
-    #rvec, tvec = cv2.solvePnPRefineVVS(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector, criteria=criteria)
-    #im_with_pose = draw_head_pose(im_with_pose, image_points, rvec, tvec, camera_matrix, dist_coeffs)
-    
-
-    #square
-    #rvec, tvec = estimate_head_pose(image_path, square_model_points,square_image_points, camera_matrix, dist_coeffs,cv2.SOLVEPNP_IPPE_SQUARE)
-    #not so good
-    #im_with_pose = draw_head_pose(im_with_pose, square_image_points, rvec, tvec, camera_matrix, dist_coeffs)
-
-    
-
-    #print(rotation_matrix)
-    # 回転行列をオイラー角に変換
-    #euler_angles = cv2.decomposeProjectionMatrix(rotation_matrix)[-1]
-
-    # オイラー角の表示 (x, y, z)
-    
-    # Display image
-    cv2.imshow("Output", cv2.cvtColor(im_with_pose, cv2.COLOR_BGR2RGB))
-    cv2.waitKey(0)
-    cv2.destroyAllWindows()
-    cv2.imwrite("result.jpg",cv2.cvtColor(im_with_pose, cv2.COLOR_BGR2RGB))
-
-if __name__ == "__main__":
-    main()

demo_header.html

@@ -1,6 +1,6 @@
 <div style="text-align: center;">
     <h1>
-        Mediapipe Face-Pose Estimation plus OpenCV
+        Mediapipe Head-Pose Estimation
     </h1>
     <div  class="grid-container">
         <img src="https://akjava.github.io/AIDiagramChatWithVoice-FaceCharacter/webp/128/00544245.webp" alt="Mediapipe Face Detection" class="image">
@@ -8,9 +8,11 @@
         <p class="text">
             This Space use <a href="http://www.apache.org/licenses/LICENSE-2.0">the Apache 2.0</a> Licensed <a href="https://ai.google.dev/edge/mediapipe/solutions/vision/face_landmarker">Mediapipe FaceLandmarker</a> <br>
             "Current MediaPipe face-landmark detection struggle with faces rotated more than 45 degrees (due to limitations in training data).<br>
-            A new tool or method is needed to achieve improved accuracy."<br>
-            OpenCV:I've tried it out, but there's still room for improvement in how the features work together.<br>
-            TODO:change base-model
+            
+            Accurate detection in MediaPipe requires correct positioning, but I don't know the exact position needed. This is a known <a href="https://github.com/google-ai-edge/mediapipe/issues/4759">issue</a><br>
+            I start to test hgbr and lgbm models,this estimate result help to make aligned image.<br>
+            center index see <a href="https://github.com/google-ai-edge/mediapipe/blob/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png">mediapipe face index image</a><br>
+            Recently Choose center index and mediapipe-estimate + models trained with face-features extracted by mediapipe
         </p>
     </div>
     

glibvision

@@ -0,0 +1 @@
+C:/Users/owner/Documents/pythons/glibvision/glibvision

glibvision/common_utils.py

@@ -1,112 +0,0 @@
-import os
-def check_exists_files(files,dirs,exit_on_error=True):
-    if files is not None:
-        if isinstance(files, str):  
-            files = [files]
-        for file in files:
-            if not os.path.isfile(file):
-                print(f"File {file} not found")
-                if exit_on_error:
-                    exit(1)
-                else:
-                    return 1
-    if dirs is not None:
-        if isinstance(dirs, str):  
-            dirs = [dirs]
-        for dir in dirs:
-            if not os.path.isdir(dir):
-                print(f"Dir {dir} not found")
-                if exit_on_error:
-                    exit(1)
-                else:
-                    return 1
-    return 0
-
-image_extensions =[".jpg"]
-
-def add_name_suffix(file_name,suffix,replace_suffix=False):
-    if not suffix.startswith("_"):#force add
-        suffix="_"+suffix
-
-    name,ext = os.path.splitext(file_name)
-    if replace_suffix:
-        index = name.rfind("_")
-        if index!=-1:
-            return f"{name[0:index]}{suffix}{ext}"
-        
-    return f"{name}{suffix}{ext}"
-
-def replace_extension(file_name,new_extension,suffix=None,replace_suffix=False):
-    if not new_extension.startswith("."):
-        new_extension="."+new_extension
-
-    name,ext = os.path.splitext(file_name)
-    new_file = f"{name}{new_extension}"
-    if suffix:
-        return add_name_suffix(name+new_extension,suffix,replace_suffix)
-    return new_file
-
-def list_digit_images(input_dir,sort=True):
-    digit_images = []
-    global image_extensions
-    files = os.listdir(input_dir)
-    for file in files:
-        if file.endswith(".jpg"):#TODO check image
-            base,ext = os.path.splitext(file)
-            if not base.isdigit():
-                continue
-            digit_images.append(file) 
-
-    if sort:
-        digit_images.sort()
-
-    return digit_images
-def list_suffix_images(input_dir,suffix,is_digit=True,sort=True):
-    digit_images = []
-    global image_extensions
-    files = os.listdir(input_dir)
-    for file in files:
-        if file.endswith(".jpg"):#TODO check image
-            base,ext = os.path.splitext(file)
-            if base.endswith(suffix):
-                if is_digit:
-                    if not base.replace(suffix,"").isdigit():
-                        continue
-                digit_images.append(file) 
-
-    if sort:
-        digit_images.sort()
-
-    return digit_images
-
-import time
-
-class ProgressTracker:
-    """
-    処理の進捗状況を追跡し、経過時間と残り時間を表示するクラス。
-    """
-
-    def __init__(self,key, total_target):
-        """
-        コンストラクタ
-
-        Args:
-            total_target (int): 処理対象の総数
-        """
-        self.key = key
-        self.total_target = total_target
-        self.complete_target = 0
-        self.start_time = time.time()
-
-    def update(self):
-        """
-        進捗を1つ進める。
-        経過時間と残り時間を表示する。
-        """
-        self.complete_target += 1
-        current_time = time.time()
-        consumed_time = current_time - self.start_time
-        remain_time = (consumed_time / self.complete_target) * (self.total_target - self.complete_target) if self.complete_target > 0 else 0 
-        print(f"stepped {self.key} {self.total_target} of {self.complete_target}, consumed {(consumed_time / 60):.1f} min, remain {(remain_time / 60):.1f} min")
-
-    

glibvision/cv2_utils.py

@@ -1,175 +0,0 @@
-import cv2
-import numpy as np
-
-#2024-11-27 add copy image
-
-def draw_bbox(image,box,color=(255,0,0),thickness=1):
-  if thickness==0:
-    return
-  
-  left = int(box[0])
-  top = int(box[1])
-  right = int(box[0]+box[2])
-  bottom = int(box[1]+box[3])
-  box_points =[(left,top),(right,top),(right,bottom),(left,bottom)]
-  
-  cv2.polylines(image, [np.array(box_points)], isClosed=True, color=color, thickness=thickness)
-
-
-def to_int_points(points):
-  int_points=[]
-  for point in points:
-    int_points.append([int(point[0]),int(point[1])])
-  return int_points
-
-def draw_text(img, text, point, font_scale=0.5, color=(200, 200, 200), thickness=1):
-  font = cv2.FONT_HERSHEY_SIMPLEX
-  cv2.putText(img, str(text), point, font, font_scale, color, thickness, cv2.LINE_AA)
-
-plot_text_color = (200, 200, 200)
-plot_text_font_scale = 0.5
-plot_index = 1
-plot_text = True
-
-def set_plot_text(is_plot,text_font_scale,text_color):
-  global plot_index,plot_text,plot_text_font_scale,plot_text_color
-  plot_text = is_plot
-  plot_index = 1
-  plot_text_font_scale = text_font_scale
-  plot_text_color = text_color
-
-def plot_points(image,points,isClosed=False,circle_size=3,circle_color=(255,0,0),line_size=1,line_color=(0,0,255)):
-    
-    global plot_index,plot_text
-    int_points = to_int_points(points)
-    if circle_size>0:
-      for point in int_points:
-        cv2.circle(image,point,circle_size,circle_color,-1)
-        if plot_text:
-          draw_text(image,plot_index,point,plot_text_font_scale,plot_text_color)
-        plot_index+=1
-    if line_size>0:
-      cv2.polylines(image, [np.array(int_points)], isClosed=isClosed, color=line_color, thickness=line_size)
-
-def fill_points(image,points,thickness=1,line_color=(255,255,255),fill_color = (255,255,255)):
-    np_points = np.array(points,dtype=np.int32)
-    cv2.fillPoly(image, [np_points], fill_color)
-    cv2.polylines(image, [np_points], isClosed=True, color=line_color, thickness=thickness)
-
-def get_image_size(cv2_image):
-    return cv2_image.shape[:2]
-
-def get_channel(np_array):
-    return np_array.shape[2] if np_array.ndim == 3 else 1 
-
-def get_numpy_text(np_array,key=""):
-    channel = get_channel(np_array)
-    return f"{key} shape = {np_array.shape} channel = {channel} ndim = {np_array.ndim} size = {np_array.size}"
-
-
-def gray3d_to_2d(grayscale: np.ndarray) -> np.ndarray:
-    channel = get_channel(grayscale)
-    if channel!=1:
-        raise ValueError(f"color maybe rgb or rgba {get_numpy_text(grayscale)}")
-    """
-    3 次元グレースケール画像 (チャンネル数 1) を 2 次元に変換する。
-
-    Args:
-        grayscale (np.ndarray): 3 次元グレースケール画像 (チャンネル数 1)。
-
-    Returns:
-        np.ndarray: 2 次元グレースケール画像。
-    """
-
-    if grayscale.ndim == 2:
-        return grayscale
-    return np.squeeze(grayscale)
-
-def blend_rgb_images(image1: np.ndarray, image2: np.ndarray, mask: np.ndarray) -> np.ndarray:
-    """
-    2 つの RGB 画像をマスク画像を使用してブレンドする。
-
-    Args:
-        image1 (np.ndarray): 最初の画像 (RGB)。
-        image2 (np.ndarray): 2 番目の画像 (RGB)。
-        mask (np.ndarray): マスク画像 (グレースケール)。
-
-    Returns:
-        np.ndarray: ブレンドされた画像 (RGB)。
-
-    Raises:
-        ValueError: 入力画像の形状が一致しない場合。
-    """
-
-    if image1.shape != image2.shape or image1.shape[:2] != mask.shape:
-        raise ValueError("入力画像の形状が一致しません。")
-
-    # 画像を float 型に変換
-    image1 = image1.astype(float)
-    image2 = image2.astype(float)
-
-    # マスクを 3 チャンネルに変換し、0-1 の範囲にスケール
-    alpha = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR).astype(float) / 255.0
-
-    # ブレンド計算
-    blended = (1 - alpha) * image1 + alpha * image2
-
-    return blended.astype(np.uint8)
-
-def create_color_image(img,color=(255,255,255)):
-    mask = np.zeros_like(img)
-    
-    h, w = img.shape[:2]
-    cv2.rectangle(mask, (0, 0), (w, h), color, -1)
-    return mask
-
-def pil_to_bgr_image(image):
-  np_image = np.array(image, dtype=np.uint8)
-  if np_image.shape[2] == 4:
-      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_RGBA2BGRA)
-  else:
-      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)
-  return bgr_img
-
-def bgr_to_rgb(np_image):
-  if np_image.shape[2] == 4:
-      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_RBGRA2RGBA)
-  else:
-      bgr_img = cv2.cvtColor(np_image, cv2.COLOR_BGR2RGB)
-  return bgr_img
-
-def copy_image(img1: np.ndarray, img2: np.ndarray, x: int, y: int) -> None:
-    # チャネル数と次元数のチェック
-    if img1.ndim != 3 or img2.ndim != 3:
-        raise ValueError("Both img1 and img2 must be 3-dimensional arrays.")
-    elif img1.shape[2] != img2.shape[2]:
-        raise ValueError(f"img1 and img2 must have the same number of channels. img1 has {img1.shape[2]} channels, but img2 has {img2.shape[1]} channels.")
-
-    # Type check
-    if not isinstance(img1, np.ndarray) or not isinstance(img2, np.ndarray):
-        raise TypeError("img1 and img2 must be NumPy arrays.")
-
-    if x>=0:
-      offset_x=0
-      w = min(img1.shape[1]-x,img2.shape[1])
-    else:
-      w = min(img1.shape[1],img2.shape[1]+x)
-      offset_x=int(-x)
-      x = 0
-
-    if y>=0:
-      h = min(img1.shape[0]-y,img2.shape[0])
-      offset_y=0
-    else:
-      h = min(img1.shape[0]-y,img2.shape[0]+y)
-      offset_y=int(-y)
-      y = 0
-    x=int(x)
-    y=int(y)
-    h=int(h)
-    w=int(w)
-   
-    
-    print(f"img1 {img1.shape} img2{img2.shape} x={x} y={y} w={w} h={h}")
-    # Paste the overlapping part
-    img1[y:y+h, x:x+w] = img2[offset_y:h+offset_y, offset_x:w+offset_x]

glibvision/draw_utils.py

@@ -1,42 +0,0 @@
-# DrawUtils 
-# not PIL,CV2,Numpy drawing method
-import math
-# 2024-11-29 add calculate_distance
-def points_to_box(points):
-  x1=float('inf')
-  x2=0
-  y1=float('inf')
-  y2=0
-  for point in points:
-    if point[0]<x1:
-      x1=point[0]
-    if point[0]>x2:
-      x2=point[0]
-    if point[1]<y1:
-      y1=point[1]
-    if point[1]>y2:
-      y2=point[1]
-  return [x1,y1,x2-x1,y2-y1]
-
-def box_to_point(box):
-  return [
-    [box[0],box[1]], 
-    [box[0]+box[2],box[1]],
-    [box[0]+box[2],box[1]+box[3]],
-    [box[0],box[1]+box[3]]
-  ]
-
-def plus_point(base_pt,add_pt):
-  return [base_pt[0]+add_pt[0],base_pt[1]+add_pt[1]]
-
-def box_to_xy(box):
-  return [box[0],box[1],box[2]+box[0],box[3]+box[1]]
-
-def to_int_points(points):
-  int_points=[]
-  for point in points:
-    int_points.append([int(point[0]),int(point[1])])
-  return int_points
-
-def calculate_distance(xy, xy2):
-    return math.sqrt((xy2[0] - xy[0])**2 + (xy2[1] - xy[1])**2)

glibvision/glandmark_utils.py

@@ -1,48 +0,0 @@
-
-import os
-
-#simple single version
-def bbox_to_glandmarks(file_name,bbox,points = None):
-     base,ext = os.path.splitext(file_name)
-     glandmark = {"image":{
-            "boxes":[{
-                "left":int(bbox[0]),"top":int(bbox[1]),"width":int(bbox[2]),"height":int(bbox[3])
-            }],
-            "file":file_name,
-            "id":int(base) 
-            # width,height ignore here
-            }}
-     if points is not None:
-          parts=[
-          ]
-          for point in points:
-               parts.append({"x":int(point[0]),"y":int(point[1])})
-          glandmark["image"]["boxes"][0]["parts"] = parts
-     return glandmark
-
-#technically this is not g-landmark/dlib ,
-def convert_to_landmark_group_json(points):
-     if len(points)!=68:
-          print(f"points must be 68 but {len(points)}")
-          return None
-     new_points=list(points)
-     
-     result = [ # possible multi person ,just possible any func support multi person
-         
-          {    # index start 0 but index-number start 1
-               "chin":new_points[0:17],
-               "left_eyebrow":new_points[17:22],
-               "right_eyebrow":new_points[22:27],
-               "nose_bridge":new_points[27:31],
-               "nose_tip":new_points[31:36],
-               "left_eye":new_points[36:42],
-               "right_eye":new_points[42:48],
-               
-               # lip points customized structure  
-               # MIT licensed face_recognition
-               # https://github.com/ageitgey/face_recognition
-               "top_lip":new_points[48:55]+[new_points[64]]+[new_points[63]]+[new_points[62]]+[new_points[61]]+[new_points[60]],
-               "bottom_lip":new_points[54:60]+[new_points[48]]+[new_points[60]]+[new_points[67]]+[new_points[66]]+[new_points[65]]+[new_points[64]],
-          }
-     ]
-     return result

glibvision/numpy_utils.py

@@ -1,110 +0,0 @@
-import numpy as np
-
-
-def apply_binary_mask_to_color(base_image,color,mask):
-    """
-    二値マスクを使用して、画像の一部を別の画像にコピーする。
-
-    Args:
-        base_image (np.ndarray): コピー先の画像。
-        paste_image (np.ndarray): コピー元の画像。
-        mask (np.ndarray): 二値マスク画像。
-
-    Returns:
-        np.ndarray: マスクを適用した画像。
-
-    """
-    # TODO check all shape
-    #print_numpy(base_image)
-    #print_numpy(paste_image)
-    #print_numpy(mask)
-    if mask.ndim == 2:
-        condition = mask == 255
-    else:
-        condition = mask[:,:,0] == 255
-    
-    base_image[condition] = color
-    return base_image
-
-def apply_binary_mask_to_image(base_image,paste_image,mask):
-    """
-    二値マスクを使用して、画像の一部を別の画像にコピーする。
-
-    Args:
-        base_image (np.ndarray): コピー先の画像。
-        paste_image (np.ndarray): コピー元の画像。
-        mask (np.ndarray): 二値マスク画像。
-
-    Returns:
-        np.ndarray: マスクを適用した画像。
-
-    """
-    # TODO check all shape
-    #print_numpy(base_image)
-    #print_numpy(paste_image)
-    #print_numpy(mask)
-    if mask.ndim == 2:
-        condition = mask == 255
-    else:
-        condition = mask[:,:,0] == 255
-    
-    base_image[condition] = paste_image[condition]
-    return base_image
-
-def pil_to_numpy(image):
-    return np.array(image, dtype=np.uint8)
-
-def extruce_points(points,index,ratio=1.5):
-    """
-    indexのポイントをratio倍だけ、点群の中心から、外側に膨らます。
-    """
-    center_point = np.mean(points, axis=0)
-    if index < 0 or index > len(points):
-        raise ValueError(f"index must be range(0,{len(points)} but value = {index})")
-    point1 =points[index]
-    print(f"center = {center_point}")
-    vec_to_center = point1 - center_point
-    return vec_to_center*ratio + center_point
-
-
-def bulge_polygon(points, bulge_factor=0.1,isClosed=True):
-    """
-    ポリゴンの辺の中間に点を追加し、外側に膨らませる
-    ndarrayを返すので注意
-    """
-    # 入力 points を NumPy 配列に変換
-    points = np.array(points)
-
-    # ポリゴン全体の重心を求める
-    center_point = np.mean(points, axis=0)
-    #print(f"center = {center_point}")
-    new_points = []
-    num_points = len(points)
-    for i in range(num_points):
-        if i == num_points -1 and not isClosed:
-            break
-        p1 = points[i]
-        #print(f"p{i} = {p1}")
-        # 重心から頂点へのベクトル
-        #vec_to_center = p1 - center_point
-        
-        # 辺のベクトルを求める
-        mid_diff = points[(i + 1) % num_points] - p1
-        mid = p1+(mid_diff/2)
-
-        #print(f"mid = {mid}")
-        out_vec = mid - center_point
-
-        # 重心からのベクトルに bulge_vec を加算
-        new_point = mid + out_vec * bulge_factor
-        
-        new_points.append(p1)
-        new_points.append(new_point.astype(np.int32))
-
-    return np.array(new_points)
-
-
-# image.shape rgb are (1024,1024,3) use 1024,1024 as 2-dimensional
-def create_2d_image(shape):
-    grayscale_image = np.zeros(shape[:2], dtype=np.uint8)
-    return grayscale_image

glibvision/pil_utils.py

@@ -1,35 +0,0 @@
-from PIL import Image,ImageDraw
-from .draw_utils import box_to_xy,to_int_points,box_to_point
-#ver-2024-11-18
-def create_color_image(width, height, color=(255,255,255)):
-    if color == None:
-        color = (0,0,0)
-    
-    if len(color )== 3:
-        mode ="RGB"
-    elif len(color )== 4:
-        mode ="RGBA"
-
-    img = Image.new(mode, (width, height), color)
-    return img
-
-# deprecated
-def fill_points(image,points,color=(255,255,255)):
-    return draw_points(image,points,fill=color)
-
-def draw_points(image,points,outline=None,fill=None,width=1):
-    
-    draw = ImageDraw.Draw(image)
-    int_points = [(int(x), int(y)) for x, y in points]
-
-    if outline is not  None or fill is not None:
-        draw.polygon(int_points, outline=outline,fill=fill,width=width)
-    
-    return image
-
-def draw_box(image,box,outline=None,fill=None):
-    points = to_int_points(box_to_point(box))
-    return draw_points(image,points,outline,fill)
-
-def from_numpy(numpy_array):
-    return Image.fromarray(numpy_array)

models/hyper-hgr-random15.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61faed39a9175818ce419775a85cc8ad3a32710f770a4f235221f280c3098876
+size 14615903

models/hyper-hgr-random15_polynomial_features.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29f0cb0ea2b44f42eac4270e913617693fcb41dca950ca86417f27f450169c71
+size 361

models/hyper-hgr-random15_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d64692adb1ab4593f035b7fcbd1d15ce64f6d0196046c9a639ac40a42355ae67
+size 43881

models/hyper-hgr-random15_selectkbest.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a150b731677290056735baf204c9abb0287676f5235e416740e9cdfa6a64fceb
+size 29337

models/hyper-hgr-random45.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03b1ceb2629ddbbc52019fb8bf90279d211938da545bd0fce34575080ef70cf
+size 14615903

models/hyper-hgr-random45_polynomial_features.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29f0cb0ea2b44f42eac4270e913617693fcb41dca950ca86417f27f450169c71
+size 361

models/hyper-hgr-random45_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7b8b0f3b8be8386ef5fb95ce41d7df962138fd894943c97cca2b2fada49d9d5
+size 43881

models/hyper-hgr-random45_selectkbest.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b76c0e458602bf9a2a99e8293cf8b370aab3b6ba6daaf326df434a547504283
+size 29337

models/hyper-hgr-random90.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abf1f0844f355860e7d0a16c13900f52e30414b7a592b3e72e7083baceb32390
+size 12316183

models/hyper-hgr-random90_polynomial_features.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29f0cb0ea2b44f42eac4270e913617693fcb41dca950ca86417f27f450169c71
+size 361

models/hyper-hgr-random90_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5086bcd611dc93afbbc2f48b98d34d955da0de4820c920f897830850b852048
+size 43881

models/hyper-hgr-random90_selectkbest.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6065c99e9919125705f0c7f5867c8dfa5d90869aff3cf035dbc3ddfb8431478d
+size 29337

models/lgbm-optimizer_15_random.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a3c7a8f13513e82a58246e15d16117bca71eebaf42aca0b02da952b0945edc6
+size 6815952

models/lgbm-optimizer_15_random_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29d9c78081a6842483621fdde85114e65aed8e667787bf7e37e1acb8bad12149
+size 2857

models/lgbm-optimizer_15dart_random.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd7347ddc3384d0f031602ba12d74dedd0ef642711bf52f38d575ef678c77765
+size 5635161

models/lgbm-optimizer_15dart_random_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be8bee58658969f1c8d94d00301ab7d2e0654f11a8a66cfaf7c71ce91f07074c
+size 2841

models/lgbm-optimizer_45_random.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9bf95c6b1dd0008987431ec470ab92e039155d89aa72ed271f478a38fc4374e
+size 6516617

models/lgbm-optimizer_45_random_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:701b13f7141d30b4ceec3e79b66f7e23da524e82c515e00120c3c229e6afbc58
+size 3177

models/lgbm-optimizer_90_random.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c18042d853049aef5d754b89cfd09ef2512303ec9b17326b99548287c29d8efe
+size 6653496

models/lgbm-optimizer_90_random_scaler.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c185a859763906f4a6fc67e8855228a20e6ab44aa54aad6460115b16db614e0a
+size 3401

mp_estimate.py

@@ -0,0 +1,253 @@
+#2024-12-04 add forehead_chin_points_pair,estimate_rotatios
+#formart is first,second,middle
+#2024-12-05 deg to rad
+#2024-12-06 get_feature_ratios_cordinate
+#2024-12-08 create_detail_labels
+#2024-12-14 ratio support 4point
+horizontal_points_pair = [
+        [
+            "inner-eye",133,362,6
+        ],
+        [
+            "outer-eye",33,263,168
+        ],
+        [
+            "mouth",61,291,13
+        ],
+         [
+            "eyeblow",105,334,9
+        ],[
+            "nose",98,327,2
+        ],[
+            "contour",143,372,6
+        ],
+        [
+            "chin",32,262,200
+        ], [
+            "cheek",123,352,5
+        ], [
+            "cheek2",192,416,0
+        ], [
+            "nose1",129,358,1
+        ], [
+            "nose2",47,277,195
+        ], [
+            "cheek3",206,426,2
+        ], [
+            "cheek4",101,330,5
+        ], [
+            "cheek5",153,380,6
+        ]
+    ]
+def angle_between_points_and_x_axis(A, B):
+        """
+        2点A, Bを結ぶ線分とx軸の正方向との角度を計算する
+
+        Args:
+            A: A点の座標 (x, y) のタプルまたはNumPy配列
+            B: B点の座標 (x, y) のタプルまたはNumPy配列
+
+        Returns:
+            角度（ラジアン）
+        """
+        x = B[0] - A[0]
+        y = B[1] - A[1]
+        return np.arctan2(y, x)
+
+vertical_points_pair=[
+    ["forehead-chin",8,1,199]
+]
+#formart is first,second,third
+feature_ratios_indices=[
+        ["forehead",67,69,66],
+        ["forehead",10,151,9],
+        ["forehead",297,299,296],
+         #["forehead-chin",8,1,199],
+         #["middle-chin",168,199,2],
+        ["middle",168,195,2],
+        ["right",153,101,206],
+        ["right2",133,47,129],
+        ["left",380,330,426],
+        ["left2",362,277,358],
+        ["right-contour",143,123,192],
+        ["left-contour",372,352,416],
+         ["nose",4,1,2],
+    ]
+
+feature_angles_indices =[
+    ["forehead1",9,6],
+    ["forehead2",69,299],
+    ["eyes1",133,362],
+    ["eyes2",133,33],
+    ["eyes3",362,263],
+    ["nose1",6,2],
+    ["nose1",98,327],
+    ["nose1",2,1],
+    ["nose1",1,6],
+    ["lip",61,291],
+    ["lip",0,17],
+    ["jaw",152,199],
+    ["jaw",194,418],
+    ["cheek",118,214],
+    ["cheek",347,434],
+    ["contour",389,397],
+     ["contour",127,172],
+]
+def get_feature_angles_cordinate(face_landmarks,angles=feature_angles_indices):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return get_feature_angles_cordinate_points(points,angles)
+
+def get_feature_angles_cordinate_points(points,angles=feature_angles_indices):
+    cordinates=[]
+    result_angles = []
+    for indices in angles:
+        points_cordinate = get_points_by_indices(points,indices[1:])#first one is label
+        angle_rad =angle_between_points_and_x_axis(points_cordinate[0][:2],points_cordinate[1][:2])
+        result_angles.append(angle_rad)
+        cordinates.append(points_cordinate)
+    return cordinates,result_angles
+
+def get_feature_ratios_cordinate(face_landmarks,ratios=feature_ratios_indices):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return get_feature_angles_cordinate_points(points,ratios)
+
+def ratios_cordinates(cordinates):
+    
+    distance_a = calculate_distance(cordinates[0],cordinates[1])
+    print(distance_a)
+    distance_b = calculate_distance(cordinates[-2],cordinates[-1])
+    print(distance_b)
+    if distance_a == 0 or distance_b == 0:
+        return 0
+    else:
+        return distance_a/distance_b
+    
+def get_feature_ratios_cordinate_points(points,ratios=feature_ratios_indices):
+    cordinates=[]
+    result_ratios = []
+    for indices in ratios:
+        points_cordinate = get_points_by_indices(points,indices[1:])#first one is label
+        result_ratios.append(ratios_cordinates(points_cordinate))
+        cordinates.append(points_cordinate)
+    return cordinates,result_ratios
+
+
+#vertical-format
+forehead_chin_points_pair=[
+    [
+        "forehead-chin",8,1,199
+    ]
+]
+horizontal_contour_points_pair=[
+    [
+        "contour",143,6,372
+    ]
+]
+import math
+def calculate_distance(xy, xy2):
+    return math.sqrt((xy2[0] - xy[0])**2 + (xy2[1] - xy[1])**2)
+
+def create_detail_labels(values,radian=False,pair_data=horizontal_points_pair):
+    assert len(values) == len(pair_data)
+    lines = []
+    for i,value in enumerate(values):
+        if radian:
+            value=math.degrees(value)
+        lines.append(f"{pair_data[i][0]} = {value:.2f}")
+    return "\n".join(lines)
+
+import numpy as np
+from mp_utils import get_normalized_cordinate
+def estimate_horizontal(face_landmarks,pair_data = horizontal_points_pair):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return estimate_horizontal_points(points,pair_data)
+
+def get_points_by_indices(face_landmark_points,indices):
+    points = [face_landmark_points[index] for index in indices]
+    return points
+
+def normalized_to_pixel(cordinates,width,height):
+    pixel_point = [[pt[0]*width,pt[1]*height] for pt in cordinates]
+    return pixel_point
+
+def estimate_horizontal_points(face_landmark_points,pair_data = horizontal_points_pair):
+    z_angles=[]
+    y_ratios = []
+    cordinates = []
+    for compare_point in pair_data:
+        points_cordinate = get_points_by_indices(face_landmark_points,compare_point[1:])#first one is label
+        cordinates.append(points_cordinate)
+        angle_rad =angle_between_points_and_x_axis(points_cordinate[0][:2],points_cordinate[1][:2])
+        #angle_deg = np.degrees(angle_rad)
+        z_angles.append(angle_rad)
+        right_distance = calculate_distance(points_cordinate[0],points_cordinate[2])
+        left_distance = calculate_distance(points_cordinate[1],points_cordinate[2])
+        y_ratios.append(left_distance/(right_distance+left_distance))
+    return z_angles,y_ratios,cordinates,pair_data
+
+def estimate_vertical(face_landmarks,pair_data = vertical_points_pair):
+    points = [get_normalized_cordinate(face_landmarks,i) for i in range(468)]
+    return estimate_vertical_points(points,pair_data)
+
+
+def estimate_rotations_v2(face_landmarker_result):
+    points = get_normalized_landmarks(face_landmarker_result.face_landmarks,True)
+    values1_text=estimate_rotations_point(points)
+    result3,ratios = get_feature_ratios_cordinate_points(points)
+    key_cordinates,angles = get_feature_angles_cordinate_points(points)
+    angles_str=[str(angle) for angle in angles]
+    ratios_str=[str(ratio) for ratio in ratios]            
+    return f"{values1_text},{','.join(angles_str)},{','.join(ratios_str)}"
+    
+from mp_utils import get_normalized_landmarks 
+def estimate_rotations(face_landmarker_result):
+    points = get_normalized_landmarks(face_landmarker_result.face_landmarks,True)
+    return estimate_rotations_point(points)
+def estimate_rotations_point(points):
+    z_angles,y_ratios,h_cordinates,_ =estimate_horizontal_points(points)
+    z_angle = np.mean(z_angles)
+    y_ratio = np.mean(y_ratios)
+    _,x_ratios,h_cordinates,_ =estimate_vertical_points(points)
+    x_ratio = np.mean(x_ratios)
+
+    x_angle,_,_,_ =estimate_vertical_points(points,forehead_chin_points_pair)
+    x_angle=np.mean(x_angle)
+    
+    length_ratio = estimate_ratio(points)
+
+    result = f"{x_ratio:.6f},{y_ratio:.6f},{z_angle:.6f},{x_angle:.6f},{length_ratio:.6f}"
+    return result
+
+def estimate_ratio(face_landmark_points,a_line=forehead_chin_points_pair,b_line=horizontal_contour_points_pair):
+    points_cordinate_a = get_points_by_indices(face_landmark_points,a_line[0][1:])#for campatible
+    points_cordinate_b = get_points_by_indices(face_landmark_points,b_line[0][1:])
+    
+    distance_a = calculate_distance(points_cordinate_a[0],points_cordinate_a[2])
+    distance_b = calculate_distance(points_cordinate_b[0],points_cordinate_b[2])
+    if distance_a == 0 or distance_b == 0:
+        return 0
+    else:
+        return distance_a/distance_b
+
+def estimate_vertical_points(face_landmarks,pair_data = vertical_points_pair):
+    angles = []
+    ratios = []
+    cordinates = []
+    for compare_point in pair_data:
+        points_cordinate = get_points_by_indices(face_landmarks,compare_point[1:])#first one is label
+        cordinates.append(points_cordinate)
+        angle_rad =angle_between_points_and_x_axis(points_cordinate[0][:2],points_cordinate[2][:2])
+        #angle_deg = np.degrees(angle_rad)
+        angles.append(angle_rad)
+        up_distance = calculate_distance(points_cordinate[0],points_cordinate[1])
+        down_distance = calculate_distance(points_cordinate[1],points_cordinate[2])
+        ratios.append(down_distance/(down_distance+up_distance))
+    return angles,ratios,cordinates,pair_data
+def mean_std_label(values,radian=False):
+    mean_value = np.mean(values)
+    std_value = np.std(values)
+    if radian:
+        mean_value = math.degrees(mean_value)
+        std_value = math.degrees(std_value)
+    value_text = f"mean:{mean_value:.3f} std:{std_value:.3f}"
+    return value_text

mp_utils.py

@@ -10,11 +10,16 @@ import numpy as np
 # 2024-11-27 -extract_landmark :add args 
 # add get_pixel_xyz
 # 2024-11-28 add get_normalized_xyz
+# 2024-11-30 add get_normalized_landmarks,sort_triangles_by_depth
+# 2024-12-04 add get_normalized_landmarks args
 def calculate_distance(p1, p2):
   """
 
   """
   return math.sqrt((p2[0] - p1[0])**2 + (p2[1] - p1[1])**2)
+
+
+
 def to_int_points(points):
     ints=[]
     for pt in points:
@@ -102,6 +107,24 @@ def get_normalized_xyz(face_landmarks_list,index):
     z=face_landmarks_list[0][index].z
     return x,y,z
 
+def get_normalized_landmarks(face_landmarks_list,recentering=False,recentering_index=4,z_multiply=0.8):
+   cordinates = [get_normalized_xyz(face_landmarks_list,i) for i in range(0,468)]
+   if recentering:
+      normalized_center_point = cordinates[recentering_index]
+      offset_x = normalized_center_point[0]
+      offset_y = normalized_center_point[1]
+
+        #need aspect?
+      cordinates = [[point[0]-offset_x,point[1]-offset_y,point[2]*z_multiply] for point in cordinates]
+
+   return cordinates
+
+
+
+def sort_triangles_by_depth(landmark_points,mesh_triangle_indices):
+   assert len(landmark_points) == 468
+   mesh_triangle_indices.sort(key=lambda triangle: sum(landmark_points[index][2] for index in triangle) / len(triangle)
+                       ,reverse=True)
 # z is normalized
 def get_pixel_xyz(face_landmarks_list,landmark,width,height):
     point = get_normalized_cordinate(face_landmarks_list,landmark)
@@ -137,4 +160,5 @@ def extract_landmark(image_data,model_path="face_landmarker.task",min_face_detec
     else:
         mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np.asarray(image_data))
     face_landmarker_result = landmarker.detect(mp_image)
-    return mp_image,face_landmarker_result
+    return mp_image,face_landmarker_result
+