solve merge

core/nbv_dataset.py

@@ -7,6 +7,7 @@ from PytorchBoot.utils.log_util import Log
 import torch
 import os
 import sys
+import time
 
 sys.path.append(r"/data/hofee/project/nbv_rec/nbv_reconstruction")
 
@@ -114,8 +115,13 @@ class NBVReconstructionDataset(BaseDataset):
         except Exception as e:
             Log.error(f"Save cache failed: {e}")
 
-    def voxel_downsample_with_mask(self, pts, voxel_size):
-        pass
+    def voxel_downsample_with_mapping(self, point_cloud, voxel_size=0.003):
+        voxel_indices = np.floor(point_cloud / voxel_size).astype(np.int32)
+        unique_voxels, inverse, counts = np.unique(voxel_indices, axis=0, return_inverse=True, return_counts=True)
+        idx_sort = np.argsort(inverse)
+        idx_unique = idx_sort[np.cumsum(counts)-counts]
+        downsampled_points = point_cloud[idx_unique]
+        return downsampled_points, inverse
 
 
     def __getitem__(self, index):
@@ -129,6 +135,9 @@ class NBVReconstructionDataset(BaseDataset):
             scanned_coverages_rate,
             scanned_n_to_world_pose,
         ) = ([], [], [])
+        start_time = time.time()
+        start_indices = [0]
+        total_points = 0
         for view in scanned_views:
             frame_idx = view[0]
             coverage_rate = view[1]
@@ -150,8 +159,12 @@ class NBVReconstructionDataset(BaseDataset):
             n_to_world_trans = n_to_world_pose[:3, 3]
             n_to_world_9d = np.concatenate([n_to_world_6d, n_to_world_trans], axis=0)
             scanned_n_to_world_pose.append(n_to_world_9d)
+            total_points += len(downsampled_target_point_cloud)
+            start_indices.append(total_points)
 
 
+        end_time = time.time()
+        #Log.info(f"load data time: {end_time - start_time}")
         nbv_idx, nbv_coverage_rate = nbv[0], nbv[1]
         nbv_path = DataLoadUtil.get_path(self.root_dir, scene_name, nbv_idx)
         cam_info = DataLoadUtil.load_cam_info(nbv_path)
@@ -164,14 +177,27 @@ class NBVReconstructionDataset(BaseDataset):
         best_to_world_9d = np.concatenate(
             [best_to_world_6d, best_to_world_trans], axis=0
         )
-
-        combined_scanned_views_pts = np.concatenate(scanned_views_pts, axis=0)
-        voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_views_pts, 0.002)
-        random_downsampled_combined_scanned_pts_np = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, self.pts_num)
         
+        combined_scanned_views_pts = np.concatenate(scanned_views_pts, axis=0)
+        voxel_downsampled_combined_scanned_pts_np, inverse = self.voxel_downsample_with_mapping(combined_scanned_views_pts, 0.003)
+        random_downsampled_combined_scanned_pts_np, random_downsample_idx = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, self.pts_num, require_idx=True)
+
+        all_idx_unique = np.arange(len(voxel_downsampled_combined_scanned_pts_np))
+        all_random_downsample_idx = all_idx_unique[random_downsample_idx]
+        scanned_pts_mask = []
+        for idx, start_idx in enumerate(start_indices):
+            if idx == len(start_indices) - 1:
+                break
+            end_idx = start_indices[idx+1]
+            view_inverse = inverse[start_idx:end_idx]
+            view_unique_downsampled_idx = np.unique(view_inverse)
+            view_unique_downsampled_idx_set = set(view_unique_downsampled_idx)
+            mask = np.array([idx in view_unique_downsampled_idx_set for idx in all_random_downsample_idx])
+            scanned_pts_mask.append(mask)
         data_item = {
             "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3)
             "combined_scanned_pts": np.asarray(random_downsampled_combined_scanned_pts_np, dtype=np.float32), # Ndarray(N x 3)
+            "scanned_pts_mask": np.asarray(scanned_pts_mask, dtype=np.bool), # Ndarray(N)
             "scanned_coverage_rate": scanned_coverages_rate, # List(S): Float, range(0, 1)
             "scanned_n_to_world_pose_9d": np.asarray(scanned_n_to_world_pose, dtype=np.float32), # Ndarray(S x 9)
             "best_coverage_rate": nbv_coverage_rate, # Float, range(0, 1)
@@ -197,7 +223,9 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["scanned_n_to_world_pose_9d"] = [
                 torch.tensor(item["scanned_n_to_world_pose_9d"]) for item in batch
             ]
-            
+            collate_data["scanned_pts_mask"] = [
+                torch.tensor(item["scanned_pts_mask"]) for item in batch
+            ]
             ''' ------ Fixed Length ------ '''
             
             collate_data["best_to_world_pose_9d"] = torch.stack(
@@ -206,12 +234,14 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["combined_scanned_pts"] = torch.stack(
                 [torch.tensor(item["combined_scanned_pts"]) for item in batch]
             )
+            
             for key in batch[0].keys():
                 if key not in [
                     "scanned_pts",
                     "scanned_n_to_world_pose_9d",
                     "best_to_world_pose_9d",
                     "combined_scanned_pts",
+                    "scanned_pts_mask",
                 ]:
                     collate_data[key] = [item[key] for item in batch]
             return collate_data
@@ -227,9 +257,9 @@ if __name__ == "__main__":
     torch.manual_seed(seed)
     np.random.seed(seed)
     config = {
-        "root_dir": "/data/hofee/data/packed_preprocessed_data",
+        "root_dir": "/data/hofee/nbv_rec_part2_preprocessed",
         "source": "nbv_reconstruction_dataset",
-        "split_file": "/data/hofee/data/OmniObject3d_train.txt",
+        "split_file": "/data/hofee/data/sample.txt",
         "load_from_preprocess": True,
         "ratio": 0.5,
         "batch_size": 2,

core/pipeline.py

@@ -75,6 +75,8 @@ class NBVReconstructionPipeline(nn.Module):
 
     def forward_test(self, data):
         main_feat = self.get_main_feat(data)
+        repeat_num = data.get("repeat_num", 1)
+        main_feat = main_feat.repeat(repeat_num, 1)
         estimated_delta_rot_9d, in_process_sample = self.view_finder.next_best_view(
             main_feat
         )
@@ -90,6 +92,8 @@ class NBVReconstructionPipeline(nn.Module):
         ]  # List(B): Tensor(S x 9)
         scanned_pts_mask_batch = data["scanned_pts_mask"] # List(B): Tensor(S x N)
 
+        scanned_pts_mask_batch = data["scanned_pts_mask"] # List(B): Tensor(N)
+
         device = next(self.parameters()).device
 
         embedding_list_batch = []

core/seq_dataset.py

@@ -50,11 +50,12 @@ class SeqReconstructionDataset(BaseDataset):
                 if not os.path.exists(os.path.join(self.root_dir, scene_name)):
                     continue
                 scene_name_list.append(scene_name)
-        return scene_name_list
+        return scene_name_list    
     
     def get_scene_name_list(self):
         return self.scene_name_list
 
+
     def get_datalist(self):
         datalist = []
         total = len(self.scene_name_list)
@@ -63,11 +64,15 @@ class SeqReconstructionDataset(BaseDataset):
             scene_max_cr_idx = 0
             frame_len = DataLoadUtil.get_scene_seq_length(self.root_dir, scene_name)
 
-            for i in range(frame_len):
+            for i in range(10,frame_len):
                 path = DataLoadUtil.get_path(self.root_dir, scene_name, i)
                 pts = DataLoadUtil.load_from_preprocessed_pts(path, "npy")
+                print(pts.shape)
                 if pts.shape[0] == 0:
                     continue
+                else:
+                    break
+            print(i)
             datalist.append({
                 "scene_name": scene_name,
                 "first_frame": i,
@@ -179,9 +184,9 @@ if __name__ == "__main__":
     np.random.seed(seed)
     
     config = {
-        "root_dir": "/media/hofee/data/data/new_testset",
+        "root_dir": "/media/hofee/data/data/test_bottle/view",
         "source": "seq_reconstruction_dataset",
-        "split_file": "/media/hofee/data/data/OmniObject3d_test.txt",
+        "split_file": "/media/hofee/data/data/test_bottle/test_bottle.txt",
         "load_from_preprocess": True,
         "filter_degree": 75,
         "num_workers": 0,
@@ -189,7 +194,7 @@ if __name__ == "__main__":
         "type": namespace.Mode.TEST,
     }
     
-    output_dir = "/media/hofee/data/data/new_testset_output"
+    output_dir = "/media/hofee/data/data/test_bottle/preprocessed_dataset"
     os.makedirs(output_dir, exist_ok=True)
 
     ds = SeqReconstructionDataset(config)

core/seq_dataset_preprocessed.py

@@ -66,7 +66,7 @@ if __name__ == "__main__":
     load_from_preprocess: True
     '''
     config = {
-        "root_dir": "H:\\AI\\Datasets\\packed_test_data",
+        "root_dir": "/media/hofee/data/data/test_bottle/preprocessed_dataset",
         "source": "seq_reconstruction_dataset",
         "split_file": "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt",
         "load_from_preprocess": True,