From c018d2c26762fec10b4334eb858c728964c4e3f3 Mon Sep 17 00:00:00 2001
From: Igor Ryabchikov <i.a.ryabchikov@gmail.com>
Date: Sun, 17 Apr 2022 13:52:02 +0300
Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5?=
 =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20api?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 common/communication/messages_pb2.py      | 40 ++++++++++++++---------
 common/communication/proto/messages.proto |  2 ++
 distance/run.py                           | 27 ++++++++++-----
 docker-compose.yml                        |  2 +-
 tracking/run.py                           |  2 +-
 5 files changed, 46 insertions(+), 27 deletions(-)

diff --git a/common/communication/messages_pb2.py b/common/communication/messages_pb2.py
index c58d365..1e8152d 100644
--- a/common/communication/messages_pb2.py
+++ b/common/communication/messages_pb2.py
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   name='messages.proto',
   package='com.devbeh.data',
   syntax='proto3',
-  serialized_pb=_b('\n\x0emessages.proto\x12\x0f\x63om.devbeh.data\"\xdc\x02\n\x11PerceptionRequest\x12\x12\n\nmanager_id\x18\x01 \x01(\x05\x12\x10\n\x08video_id\x18\x02 \x01(\x03\x12\x10\n\x08\x66rame_id\x18\x03 \x01(\x03\x12\x32\n\tdetection\x18\x04 \x01(\x0b\x32\x1d.com.devbeh.data.DetectionReqH\x00\x12\x30\n\x08tracking\x18\x05 \x01(\x0b\x32\x1c.com.devbeh.data.TrackingReqH\x00\x12,\n\x06pose3d\x18\x06 \x01(\x0b\x32\x1a.com.devbeh.data.Pose3DReqH\x00\x12\x30\n\x08\x64istance\x18\x07 \x01(\x0b\x32\x1c.com.devbeh.data.DistanceReqH\x00\x12.\n\x07\x61\x63tions\x18\x08 \x01(\x0b\x32\x1b.com.devbeh.data.ActionsReqH\x00\x12\x12\n\x08\x66inished\x18\t \x01(\x08H\x00\x42\x05\n\x03req\"\xe6\x02\n\x12PerceptionResponse\x12\x10\n\x08video_id\x18\x01 \x01(\x03\x12\x10\n\x08\x66rame_id\x18\x02 \x01(\x03\x12)\n\x03\x65rr\x18\x03 \x01(\x0b\x32\x1a.com.devbeh.data.ErrorRespH\x00\x12\x33\n\tdetection\x18\x04 \x01(\x0b\x32\x1e.com.devbeh.data.DetectionRespH\x00\x12\x31\n\x08tracking\x18\x05 \x01(\x0b\x32\x1d.com.devbeh.data.TrackingRespH\x00\x12-\n\x06pose3d\x18\x06 \x01(\x0b\x32\x1b.com.devbeh.data.Pose3DRespH\x00\x12\x31\n\x08\x64istance\x18\x07 \x01(\x0b\x32\x1d.com.devbeh.data.DistanceRespH\x00\x12/\n\x07\x61\x63tions\x18\x08 \x01(\x0b\x32\x1c.com.devbeh.data.ActionsRespH\x00\x42\x06\n\x04resp\"\x1d\n\x0c\x44\x65tectionReq\x12\r\n\x05image\x18\x01 \x01(\x0c\"A\n\rDetectionResp\x12\x30\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1f.com.devbeh.data.DetectionEntry\"\xb0\x01\n\x0e\x44\x65tectionEntry\x12\x16\n\x0e\x62ox_top_left_x\x18\x01 \x01(\x05\x12\x16\n\x0e\x62ox_top_left_y\x18\x02 \x01(\x05\x12\x1a\n\x12\x62ox_bottom_right_x\x18\x03 \x01(\x05\x12\x1a\n\x12\x62ox_bottom_right_y\x18\x04 \x01(\x05\x12\x10\n\x08\x63lass_id\x18\x05 \x01(\x05\x12\r\n\x05score\x18\x06 \x01(\x01\x12\x15\n\rblurred_patch\x18\x07 \x01(\x0c\"?\n\tPose3DReq\x12\x32\n\x07\x65ntries\x18\x01 \x03(\x0b\x32!.com.devbeh.data.PoseRequestEntry\"[\n\x10PoseRequestEntry\x12\r\n\x05patch\x18\x01 \x01(\x0c\x12\x12\n\ntop_left_x\x18\x02 \x01(\x05\x12\x12\n\ntop_left_y\x18\x03 \x01(\x05\x12\x10\n\x08track_id\x18\x04 \x01(\x03\"C\n\nPose3DResp\x12\x35\n\x07\x65ntries\x18\x01 \x03(\x0b\x32$.com.devbeh.data.PoseEstimationEntry\"Q\n\x13PoseEstimationEntry\x12(\n\x06joints\x18\x01 \x03(\x0b\x32\x18.com.devbeh.data.Point3d\x12\x10\n\x08track_id\x18\x02 \x01(\x03\"*\n\x07Point3d\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"N\n\x0bTrackingReq\x12\r\n\x05image\x18\x01 \x01(\x0c\x12\x30\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x1f.com.devbeh.data.DetectionEntry\"?\n\x0cTrackingResp\x12/\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1e.com.devbeh.data.TrackingEntry\"f\n\rTrackingEntry\x12\x32\n\tdetection\x18\x01 \x01(\x0b\x32\x1f.com.devbeh.data.DetectionEntry\x12\n\n\x02id\x18\x02 \x01(\x03\x12\x15\n\rdetection_ind\x18\x03 \x01(\x05\"\xa1\x01\n\x0b\x44istanceReq\x12\r\n\x05image\x18\x01 \x01(\x0c\x12\x18\n\x10\x63\x61mera_height_mm\x18\x02 \x01(\x02\x12\x17\n\x0f\x66ocal_length_mm\x18\x03 \x01(\x02\x12\x15\n\rpixel_size_mm\x18\x04 \x01(\x02\x12\x39\n\x07\x65ntries\x18\x05 \x03(\x0b\x32(.com.devbeh.data.DistanceEstimationEntry\"C\n\x17\x44istanceEstimationEntry\x12(\n\x06points\x18\x01 \x03(\x0b\x32\x18.com.devbeh.data.Point3d\"\x8e\x01\n\x0c\x44istanceResp\x12/\n\rnormal_vector\x18\x01 \x01(\x0b\x32\x18.com.devbeh.data.Point3d\x12\x39\n\x07\x65ntries\x18\x02 \x03(\x0b\x32(.com.devbeh.data.DistanceEstimationEntry\x12\x12\n\nmask_image\x18\x03 \x01(\x0c\"]\n\nActionsReq\x12\x33\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\".com.devbeh.data.ActionPersonEntry\x12\x1a\n\x12window_size_frames\x18\x02 \x01(\x05\"I\n\x11\x41\x63tionPersonEntry\x12\n\n\x02id\x18\x01 \x01(\x03\x12(\n\x06joints\x18\x02 \x03(\x0b\x32\x18.com.devbeh.data.Point3d\"O\n\x0b\x41\x63tionsResp\x12(\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x17.com.devbeh.data.Action\x12\x16\n\x0elatency_frames\x18\x02 \x01(\x05\"K\n\x06\x41\x63tion\x12\x17\n\x0f\x66irst_person_id\x18\x01 \x01(\x03\x12\x18\n\x10second_person_id\x18\x02 \x01(\x03\x12\x0e\n\x06\x61\x63tion\x18\x03 \x01(\x05\"\x18\n\tErrorResp\x12\x0b\n\x03msg\x18\x01 \x01(\tB\x02P\x01\x62\x06proto3')
+  serialized_pb=_b('\n\x0emessages.proto\x12\x0f\x63om.devbeh.data\"\xdc\x02\n\x11PerceptionRequest\x12\x12\n\nmanager_id\x18\x01 \x01(\x05\x12\x10\n\x08video_id\x18\x02 \x01(\x03\x12\x10\n\x08\x66rame_id\x18\x03 \x01(\x03\x12\x32\n\tdetection\x18\x04 \x01(\x0b\x32\x1d.com.devbeh.data.DetectionReqH\x00\x12\x30\n\x08tracking\x18\x05 \x01(\x0b\x32\x1c.com.devbeh.data.TrackingReqH\x00\x12,\n\x06pose3d\x18\x06 \x01(\x0b\x32\x1a.com.devbeh.data.Pose3DReqH\x00\x12\x30\n\x08\x64istance\x18\x07 \x01(\x0b\x32\x1c.com.devbeh.data.DistanceReqH\x00\x12.\n\x07\x61\x63tions\x18\x08 \x01(\x0b\x32\x1b.com.devbeh.data.ActionsReqH\x00\x12\x12\n\x08\x66inished\x18\t \x01(\x08H\x00\x42\x05\n\x03req\"\xe6\x02\n\x12PerceptionResponse\x12\x10\n\x08video_id\x18\x01 \x01(\x03\x12\x10\n\x08\x66rame_id\x18\x02 \x01(\x03\x12)\n\x03\x65rr\x18\x03 \x01(\x0b\x32\x1a.com.devbeh.data.ErrorRespH\x00\x12\x33\n\tdetection\x18\x04 \x01(\x0b\x32\x1e.com.devbeh.data.DetectionRespH\x00\x12\x31\n\x08tracking\x18\x05 \x01(\x0b\x32\x1d.com.devbeh.data.TrackingRespH\x00\x12-\n\x06pose3d\x18\x06 \x01(\x0b\x32\x1b.com.devbeh.data.Pose3DRespH\x00\x12\x31\n\x08\x64istance\x18\x07 \x01(\x0b\x32\x1d.com.devbeh.data.DistanceRespH\x00\x12/\n\x07\x61\x63tions\x18\x08 \x01(\x0b\x32\x1c.com.devbeh.data.ActionsRespH\x00\x42\x06\n\x04resp\"\x1d\n\x0c\x44\x65tectionReq\x12\r\n\x05image\x18\x01 \x01(\x0c\"A\n\rDetectionResp\x12\x30\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1f.com.devbeh.data.DetectionEntry\"\xb0\x01\n\x0e\x44\x65tectionEntry\x12\x16\n\x0e\x62ox_top_left_x\x18\x01 \x01(\x05\x12\x16\n\x0e\x62ox_top_left_y\x18\x02 \x01(\x05\x12\x1a\n\x12\x62ox_bottom_right_x\x18\x03 \x01(\x05\x12\x1a\n\x12\x62ox_bottom_right_y\x18\x04 \x01(\x05\x12\x10\n\x08\x63lass_id\x18\x05 \x01(\x05\x12\r\n\x05score\x18\x06 \x01(\x01\x12\x15\n\rblurred_patch\x18\x07 \x01(\x0c\"?\n\tPose3DReq\x12\x32\n\x07\x65ntries\x18\x01 \x03(\x0b\x32!.com.devbeh.data.PoseRequestEntry\"[\n\x10PoseRequestEntry\x12\r\n\x05patch\x18\x01 \x01(\x0c\x12\x12\n\ntop_left_x\x18\x02 \x01(\x05\x12\x12\n\ntop_left_y\x18\x03 \x01(\x05\x12\x10\n\x08track_id\x18\x04 \x01(\x03\"C\n\nPose3DResp\x12\x35\n\x07\x65ntries\x18\x01 \x03(\x0b\x32$.com.devbeh.data.PoseEstimationEntry\"Q\n\x13PoseEstimationEntry\x12(\n\x06joints\x18\x01 \x03(\x0b\x32\x18.com.devbeh.data.Point3d\x12\x10\n\x08track_id\x18\x02 \x01(\x03\"*\n\x07Point3d\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"N\n\x0bTrackingReq\x12\r\n\x05image\x18\x01 \x01(\x0c\x12\x30\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x1f.com.devbeh.data.DetectionEntry\"?\n\x0cTrackingResp\x12/\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x1e.com.devbeh.data.TrackingEntry\"f\n\rTrackingEntry\x12\x32\n\tdetection\x18\x01 \x01(\x0b\x32\x1f.com.devbeh.data.DetectionEntry\x12\n\n\x02id\x18\x02 \x01(\x03\x12\x15\n\rdetection_ind\x18\x03 \x01(\x05\"\xd2\x01\n\x0b\x44istanceReq\x12\r\n\x05image\x18\x01 \x01(\x0c\x12\x18\n\x10\x63\x61mera_height_mm\x18\x02 \x01(\x02\x12\x17\n\x0f\x66ocal_length_mm\x18\x03 \x01(\x02\x12\x15\n\rpixel_size_mm\x18\x04 \x01(\x02\x12\x39\n\x07\x65ntries\x18\x05 \x03(\x0b\x32(.com.devbeh.data.DistanceEstimationEntry\x12/\n\rnormal_vector\x18\x06 \x01(\x0b\x32\x18.com.devbeh.data.Point3d\"C\n\x17\x44istanceEstimationEntry\x12(\n\x06points\x18\x01 \x03(\x0b\x32\x18.com.devbeh.data.Point3d\"\x8e\x01\n\x0c\x44istanceResp\x12/\n\rnormal_vector\x18\x01 \x01(\x0b\x32\x18.com.devbeh.data.Point3d\x12\x39\n\x07\x65ntries\x18\x02 \x03(\x0b\x32(.com.devbeh.data.DistanceEstimationEntry\x12\x12\n\nmask_image\x18\x03 \x01(\x0c\"]\n\nActionsReq\x12\x33\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\".com.devbeh.data.ActionPersonEntry\x12\x1a\n\x12window_size_frames\x18\x02 \x01(\x05\"I\n\x11\x41\x63tionPersonEntry\x12\n\n\x02id\x18\x01 \x01(\x03\x12(\n\x06joints\x18\x02 \x03(\x0b\x32\x18.com.devbeh.data.Point3d\"O\n\x0b\x41\x63tionsResp\x12(\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x17.com.devbeh.data.Action\x12\x16\n\x0elatency_frames\x18\x02 \x01(\x05\"K\n\x06\x41\x63tion\x12\x17\n\x0f\x66irst_person_id\x18\x01 \x01(\x03\x12\x18\n\x10second_person_id\x18\x02 \x01(\x03\x12\x0e\n\x06\x61\x63tion\x18\x03 \x01(\x05\"\x18\n\tErrorResp\x12\x0b\n\x03msg\x18\x01 \x01(\tB\x02P\x01\x62\x06proto3')
 )
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
@@ -687,6 +687,13 @@ _DISTANCEREQ = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
+    _descriptor.FieldDescriptor(
+      name='normal_vector', full_name='com.devbeh.data.DistanceReq.normal_vector', index=5,
+      number=6, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
   ],
   extensions=[
   ],
@@ -700,7 +707,7 @@ _DISTANCEREQ = _descriptor.Descriptor(
   oneofs=[
   ],
   serialized_start=1628,
-  serialized_end=1789,
+  serialized_end=1838,
 )
 
 
@@ -730,8 +737,8 @@ _DISTANCEESTIMATIONENTRY = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1791,
-  serialized_end=1858,
+  serialized_start=1840,
+  serialized_end=1907,
 )
 
 
@@ -775,8 +782,8 @@ _DISTANCERESP = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1861,
-  serialized_end=2003,
+  serialized_start=1910,
+  serialized_end=2052,
 )
 
 
@@ -813,8 +820,8 @@ _ACTIONSREQ = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2005,
-  serialized_end=2098,
+  serialized_start=2054,
+  serialized_end=2147,
 )
 
 
@@ -851,8 +858,8 @@ _ACTIONPERSONENTRY = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2100,
-  serialized_end=2173,
+  serialized_start=2149,
+  serialized_end=2222,
 )
 
 
@@ -889,8 +896,8 @@ _ACTIONSRESP = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2175,
-  serialized_end=2254,
+  serialized_start=2224,
+  serialized_end=2303,
 )
 
 
@@ -934,8 +941,8 @@ _ACTION = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2256,
-  serialized_end=2331,
+  serialized_start=2305,
+  serialized_end=2380,
 )
 
 
@@ -965,8 +972,8 @@ _ERRORRESP = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2333,
-  serialized_end=2357,
+  serialized_start=2382,
+  serialized_end=2406,
 )
 
 _PERCEPTIONREQUEST.fields_by_name['detection'].message_type = _DETECTIONREQ
@@ -1024,6 +1031,7 @@ _TRACKINGREQ.fields_by_name['entries'].message_type = _DETECTIONENTRY
 _TRACKINGRESP.fields_by_name['entries'].message_type = _TRACKINGENTRY
 _TRACKINGENTRY.fields_by_name['detection'].message_type = _DETECTIONENTRY
 _DISTANCEREQ.fields_by_name['entries'].message_type = _DISTANCEESTIMATIONENTRY
+_DISTANCEREQ.fields_by_name['normal_vector'].message_type = _POINT3D
 _DISTANCEESTIMATIONENTRY.fields_by_name['points'].message_type = _POINT3D
 _DISTANCERESP.fields_by_name['normal_vector'].message_type = _POINT3D
 _DISTANCERESP.fields_by_name['entries'].message_type = _DISTANCEESTIMATIONENTRY
diff --git a/common/communication/proto/messages.proto b/common/communication/proto/messages.proto
index 6185b5b..f63e05a 100644
--- a/common/communication/proto/messages.proto
+++ b/common/communication/proto/messages.proto
@@ -132,6 +132,8 @@ message DistanceReq {
   // реализации компонента детектирования расстояния, переданные координаты z могут не учитываться, и детектироваться с
   // нуля. Считается, что по крайней мере одна точка касается земли
   repeated DistanceEstimationEntry entries = 5;
+  // Может использоваться для ручного задания вектора нормали, если он известен и не меняется для определенной камеры
+  Point3d normal_vector = 6;
 }
 
 message DistanceEstimationEntry {
diff --git a/distance/run.py b/distance/run.py
index 0e85f82..206e281 100644
--- a/distance/run.py
+++ b/distance/run.py
@@ -140,11 +140,11 @@ def detect_plane(image, obj_entries, options, config, detector: PlaneRCNNDetecto
             s1 += np.sum(np.where(mask[sa_tl_y:sa_br_y, sa_tl_x:sa_br_x] == 1, sa, 0))
 
         s = s0 + s1
-        logging.info("Plane scores " + str(len(ground_plane_scores)) + " " + str(s0) + " " + str(s1))
+        # logging.info("Plane scores " + str(len(ground_plane_scores)) + " " + str(s0) + " " + str(s1))
         ground_plane_scores.append(s)
 
     ground_plane_ind = np.argmax(np.array(ground_plane_scores))
-    logging.info('Ground plane index: ' + str(ground_plane_ind))
+    # logging.info('Ground plane index: ' + str(ground_plane_ind))
 
     normal_vector = normal_vectors[ground_plane_ind]
     # Преобразуем вектор в пространство камеры - ось x - слева направо, y - сверху вниз, z - вперед от камеры
@@ -175,6 +175,11 @@ def estimate_distance(height, width, pixel_size_mm, focal_length_mm, camera_heig
         e_points[:, 0] = e_points[:, 0] - width / 2
         e_points[:, 1] = e_points[:, 1] - height / 2
 
+        # Гарантируем, что точки, расположенные точно в центре, отсутствуют
+        for i in range(e_points.shape[0]):
+            if e_points[i, 0] == 0:
+                e_points[i, 0] = 1
+
         e_points_mm = e_points * pixel_size_mm
 
         lowest_i = find_the_lowest_point(e_points_mm, normal_vector)
@@ -211,14 +216,18 @@ def process_distance_requests(requests: Iterable[PerceptionRequest], results: It
                 e_points.append([p.x, p.y, p.z])
             obj_entries.append(np.array(e_points))
 
-        if plane_info_key in cached_planes_info.keys():
-            nv, mask_image, last_frame_id = cached_planes_info[plane_info_key]
-            if req.frame_id - last_frame_id >= estimation_period_frames - 1:
-                del cached_planes_info[plane_info_key]
+        if req.distance.normal_vector.x or req.distance.normal_vector.y or req.distance.normal_vector.z:
+            nv = (req.distance.normal_vector.x, req.distance.normal_vector.y, req.distance.normal_vector.z)
+            mask_image = None
         else:
-            nv, mask_image = detector(img, obj_entries)
-            if estimation_period_frames > 1:
-                cached_planes_info[plane_info_key] = (nv, mask_image, req.frame_id)
+            if plane_info_key in cached_planes_info.keys():
+                nv, mask_image, last_frame_id = cached_planes_info[plane_info_key]
+                if req.frame_id - last_frame_id >= estimation_period_frames - 1:
+                    del cached_planes_info[plane_info_key]
+            else:
+                nv, mask_image = detector(img, obj_entries)
+                if estimation_period_frames > 1:
+                    cached_planes_info[plane_info_key] = (nv, mask_image, req.frame_id)
 
         res_obj_entries = estimate_distance(img.shape[0], img.shape[1], req.distance.pixel_size_mm,
                                             req.distance.focal_length_mm, req.distance.camera_height_mm, nv,
diff --git a/docker-compose.yml b/docker-compose.yml
index a886fa9..754a970 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -135,7 +135,7 @@ services:
         --username "${DEVBEH_KAFKA_USERNAME:-}"
         --password "${DEVBEH_KAFKA_PASSWORD:-}"
         --batch_size ${DEVBEH_TRACKING_BATCH_SIZE:-1}
-        --max_frames_lost ${DEVBEH_MAX_FRAMES_LOST:-50}
+        --max_frames_lost ${DEVBEH_MAX_FRAMES_LOST:-100}
         --name ${DEVBEH_TRACKING_INSTANCE_NAME:-tracking}'
   distance:
     # Текущей директорией в путях является директория, в которой расположен данный docker-compose.yml файл
diff --git a/tracking/run.py b/tracking/run.py
index 47bde04..4ccdf04 100644
--- a/tracking/run.py
+++ b/tracking/run.py
@@ -205,7 +205,7 @@ def create_tracker(max_frames_lost):
     args.track_buffer = max_frames_lost
     args.mot20 = False  # todo: попробовать True тоже
     args.match_thresh = 0.9
-    return BYTETracker(args, fps)
+    return BYTETracker(args, frame_rate=fps)
 
 
 def get_fairmot_tracker_factory(fairmot_weights_path, max_frames_lost):
-- 
GitLab