success

data_generation/README.md

@@ -0,0 +1,40 @@
+# Update: Get View Interface
+* First, generate a scene file by
+```sh
+# Remember to specify dataset path in the script's main function
+python data_generation/tools/generate_scene_pcd.py
+# It will generate a scene.pickle file in the dataset folder
+```
+
+* Then, try get_view function
+```sh
+# Remember to specity the scene.pickle file path and camera pose in the script's main function
+python data_generation/tools/get_view.py
+# The get_view() function will return scene_pcl and obj_pcl_dict
+```
+
+
+# Data Generation
+This folder contains assets related to generating datasets. Maintained by Zhengxiao Han.
+
+## 1 Prerequisites
+* **OmniObject3d-simplified** dataset downloaded. Contact Jiyao to acquire it.
+* [Isaac Sim](https://docs.omniverse.nvidia.com/isaacsim/latest/index.html) installed.
+
+## 2 How to Use
+
+### 2.1 Convert Dataset to .usd Format
+
+```sh
+ ~/.local/share/ov/pkg/isaac_sim-2023.1.1/python.sh <path-to-this-repo>/data_generation/tools/convert_dataset.py <path-to-the-dataset>
+
+ # For example:
+ # ~/.local/share/ov/pkg/isaac_sim-2023.1.1/python.sh /home/hzx/Projects/ActivePerception/data_generation/tools/convert_dataset.py /home/hzx/Downloads/OmniObject3d-simplified/output
+```
+
+### 2.2 Run the Script
+* Firstly Specify file paths in [data_generation.yaml](https://github.com/Jiyao06/ActivePerception/blob/main/data_generation/config/data_generation/data_generation.yaml) and [replicator.yaml](https://github.com/Jiyao06/ActivePerception/blob/main/data_generation/config/replicator/replicator.yaml)
+* Then run this script:
+```sh
+ ~/.local/share/ov/pkg/isaac_sim-2023.1.1/python.sh <path-to-this-repo>/data_generation/src/generate_objects.py
+```

data_generation/config/data_generation/data_generation.yaml

@@ -0,0 +1,29 @@
+background_path: "/home/hzx/Projects/ActivePerception/data_generation/data/backgrounds/env.usd"
+object_dataset_path: "/home/hzx/Projects/ActivePerception/data_generation/data/objects"
+occluder_dataset_path: "/home/hzx/Projects/ActivePerception/data_generation/data/occluders/table/table_001"
+data_save_path: "/home/hzx/Projects/ActivePerception/data_generation/output"
+
+# Scene Generation Params
+num_scenes: 10
+
+# Occluder Spawning Params
+distance_threshold: 0.1
+
+# Objects Spawning Params
+model_scaling : [0.001, 0.001, 0.001]
+object_num: 20
+max_spawn_z: 2.0
+spawn_z_offset: 0.025
+num_attemp_times: 50
+max_num_planes: 2
+
+# Camera Sampling Params
+min_radius: 1.4
+max_radius: 1.5
+hfov: 90.0
+start_h_angle: 0.0
+dr: 0.05
+dhfov: 30.0
+dvfov: 30.0
+look_at_position: [0, 0, 0.3]
+look_at_noise: 0.075

data_generation/config/isaac_sim/isaac_sim.yaml

@@ -0,0 +1,8 @@
+width: 1920
+height: 1080
+sync_loads: True
+headless: False
+renderer: "RayTracedLighting"
+physics_dt: 0.01 #100hz
+rendering_dt: 0.01 #100hz
+meters_unit: 1.0

data_generation/config/replicator/replicator.yaml

@@ -0,0 +1,16 @@
+writer: "BasicWriter"
+writer_config:
+  output_dir: "/home/hzx/Projects/ActivePerception/data_generation/output"
+  rgb: True
+  camera_params: True
+  bounding_box_2d_tight: False
+  semantic_segmentation: True
+  distance_to_image_plane: True
+  distance_to_camera: False
+  bounding_box_3d: False
+  occlusion: False
+                
+resolution: [640, 480]
+max_sim_steps: 50
+rt_subframes: 1
+num_frames: 1

data_generation/config/scene_generation/object_offsets.yaml

@@ -0,0 +1,74 @@
+simple_room:
+    table_001: 
+        env: 0.255
+        obj: -0.21
+    table_002:
+        env: 0.26
+        obj: -0.22
+    table_003:
+        env: 0.265
+        obj: -0.167
+    table_004:
+        env: 0.225
+        obj: -0.19
+    table_005:
+        env: 0.14
+        obj: -0.315
+    table_007:
+        env: 0.53
+        obj: -0.095
+    table_008:
+        env: 0.215
+        obj: -0.255
+    table_009:
+        env: 0.245
+        obj: -0.2
+    table_013:
+        env: 0.51
+        obj: -0.115
+    table_014:
+        env: 0.325
+        obj: -0.2
+    table_015:
+        env: 0.29
+        obj: -0.215
+    table_016:
+        env: 0.23
+        obj: -0.255
+    table_017:
+        env: 0.32
+        obj: -0.145
+    table_018:
+        env: 0.17
+        obj: -0.128
+    table_019:
+        env: 0.26
+        obj: -0.22
+    table_020:
+        env: 0.01
+        obj: -0.31
+    table_021:
+        env: 0.295
+        obj: -0.235
+    table_022:
+        env: 0.05
+        obj: -0.055
+    table_023:
+        env: 0.046
+        obj: -0.055
+    table_024:
+        env: 0.015
+        obj: -0.255
+    table_025:
+        env: 0.29
+        obj: -0.26
+    table_026:
+        env: 0.03
+        obj: -0.3
+    table_027:
+        env: 0.01
+        obj: -0.31
+    table_026:
+        env: 0.41
+        obj: -0.08
+        

data_generation/src/generate_objects.py

@@ -0,0 +1,435 @@
+import sys
+import yaml
+import omni
+from omni.isaac.kit import SimulationApp
+import carb
+from torch._C import NoneType
+import os
+import time
+import random
+import numpy as np
+import trimesh
+from sklearn import linear_model
+import copy
+import math
+from generate_objects_utils import *
+
+class GENERATE_BOJECTS():
+    def __init__(self):
+        # Initialize Parameters
+        self._read_params()
+
+        # Initialize Isaac Sim Simulator
+        self._setup_isaac_sim()
+
+        # Initialize Variables and Parameters Related to Data Generation
+        self._setup_data_generatin_params()
+
+        # Start Main Function
+        self._main()
+    
+    def _read_params(self):
+        # Load Parameters for Launching Isaac Sim
+        _isaac_sim_config_path = sys.path[0]+'/../config/isaac_sim/isaac_sim.yaml'
+        with open(_isaac_sim_config_path, 'r') as config:
+            self._isaac_sim_config = yaml.safe_load(config)
+
+        # Load Parameters for Data Generation
+        _data_generation_config_path = sys.path[0]+'/../config/data_generation/data_generation.yaml'
+        with open(_data_generation_config_path, 'r') as config:
+            self._data_generation_config = yaml.safe_load(config)
+        if(self._data_generation_config["data_save_path"][-1]=="/"):
+            self._data_generation_config["data_save_path"][-1] = self._data_generation_config["data_save_path"][:-1]
+
+        # Load Parameters for Replicator
+        _replicator_config_path = sys.path[0]+'/../config/replicator/replicator.yaml'
+        with open(_replicator_config_path, 'r') as config:
+            self._replicator_config = yaml.safe_load(config)
+        if(self._replicator_config["writer_config"]["output_dir"][-1]=="/"):
+            self._replicator_config["writer_config"]["output_dir"] = self._replicator_config["writer_config"]["output_dir"][:-1]
+
+        # Calculate Parameters
+        self._num_r = int((self._data_generation_config["max_radius"]-self._data_generation_config["min_radius"])/self._data_generation_config["dr"])
+        # carb.log_error("num_r: "+str(self._data_generation_config["max_radius"]-self._data_generation_config["min_radius"]))
+        self._num_h = int(self._data_generation_config["hfov"]/self._data_generation_config["dhfov"])
+        self._num_v = int(360/self._data_generation_config["dvfov"])
+
+    def _setup_isaac_sim(self):
+        # Start the omniverse application
+        self._simulation_app = SimulationApp(launch_config=self._isaac_sim_config)
+
+        # The following items must be imported after simulation is initialized
+        from omni.isaac.core import World
+        from omni.isaac.dynamic_control import _dynamic_control
+
+        # Acquire dynamic control interface
+        self._dc = _dynamic_control.acquire_dynamic_control_interface()
+
+        # Disable joystick to avoid conflict
+        carb.settings.get_settings().set("persistent/app/omniverse/gamepadCameraControl", False)
+
+        # Create a new world
+        self._world = World(physics_dt=self._isaac_sim_config["physics_dt"],
+                            rendering_dt=self._isaac_sim_config["rendering_dt"],
+                            stage_units_in_meters=self._isaac_sim_config["meters_unit"])
+        if(self._world is NoneType): 
+            carb.log_error("Failed to set world.")
+        self._stage = omni.usd.get_context().get_stage()
+        self._simulation_app.update()
+
+        # Start Simulation
+        omni.timeline.get_timeline_interface().play()
+
+    def _setup_replicator(self, scene_id = 0):
+        # Initialize Replicator
+        import omni.replicator.core as rep
+        self._rep = rep
+
+        # Setup Cameras
+        self._driver_cam = self._rep.create.camera(name="DriverCam")
+        self._driver_rp = self._rep.create.render_product(self._driver_cam, self._replicator_config["resolution"], name="DriverView")
+
+        # Setup Writers
+        output_path = self._replicator_config["writer_config"]["output_dir"]
+        self._replicator_config["writer_config"]["output_dir"] = output_path + "/scene_" + str(scene_id)
+        self._writer = self._rep.WriterRegistry.get(self._replicator_config["writer"])
+        self._writer.initialize(**self._replicator_config["writer_config"])
+        self._writer.attach(self._driver_rp)
+
+    def _setup_data_generatin_params(self):
+        self._trimesh_objects = []
+        self._trimesh_occluders = []
+        self._occluder_top_z = 0
+        self._occluder_spawn_x = 0
+        self._occluder_spawn_y = 0
+        self._occluder_spawn_z = 0
+        self._occluder_x_lb = 0
+        self._occluder_x_ub = 0
+        self._occluder_y_lb = 0
+        self._occluder_y_ub = 0
+
+        self._selected_occluder_index = 0
+        self._selected_object_indices = []
+
+        self._replicator_objects = []
+
+    def _extract_model_type(self, path):
+        # Specify it according to specific file directory
+        NUM_SLASH_BEFORE_TYPE = 2
+
+        num_slash = 0
+        object_type_str = []
+
+        for i in range(len(path)):
+            index = len(path) -1 - i
+            char = path[index]
+
+            if(num_slash == NUM_SLASH_BEFORE_TYPE):
+                object_type_str.append(char)
+
+            if(char == "/"):
+                num_slash += 1
+
+        object_type_str.reverse()
+        object_type_str = ''.join(object_type_str[1:])
+        return object_type_str
+
+    def _extract_model_name(self, path):
+        # Specify it according to specific file directory
+        NUM_SLASH_BEFORE_NAME = 1
+
+        num_slash = 0
+        object_name_str = []
+
+        for i in range(len(path)):
+            index = len(path) -1 - i
+            char = path[index]
+
+            if(num_slash == NUM_SLASH_BEFORE_NAME):
+                object_name_str.append(char)
+
+            if(char == "/"):
+                num_slash += 1
+
+        object_name_str.reverse()
+        object_name_str = ''.join(object_name_str[1:])
+        return object_name_str
+
+    def _add_background(self, scene_usd_path):
+        from omni.isaac.core.utils.stage import add_reference_to_stage
+        from omni.isaac.core.prims import XFormPrim
+        add_reference_to_stage(usd_path=scene_usd_path, prim_path="/World/Env")
+        self._world.scene.add(XFormPrim(prim_path="/World/Env", name=scene_usd_path, 
+                                        position=[0,0,0], orientation=[0,0,0,1], scale=[1,1,1]))
+        
+    def _set_background_position(self, position):
+        _object_prim = self._stage.GetPrimAtPath("/World/Env")
+        _object_prim.GetAttribute("xformOp:translate").Set((position[0], position[1], position[2]))
+
+    def _delete_background(self):
+        import omni.isaac.core.utils.prims as prims_utils
+        prims_utils.delete_prim("/World/Env")
+
+    def _add_object_to_sim(self, obj_index):
+        import omni.replicator.core as rep
+
+        # Load Object
+        trimesh_obj = self._trimesh_objects[obj_index]
+
+        # Calculate proper spawn pose
+        bounding_box = trimesh_obj.bb
+        # spawn_z = abs(0-bounding_box[0][1]) + self._occluder_top_z + 0.01
+        
+        convex_hulls = self._trimesh_occluders[self._selected_occluder_index].convex_hulls
+        num_planes = len(convex_hulls)
+        if(num_planes >= self._data_generation_config["max_num_planes"]):
+            num_planes = self._data_generation_config["max_num_planes"]
+        random_plane_index = random.randint(0, num_planes-1)
+        convex_hull = convex_hulls[random_plane_index]
+
+        # calculate spawn height
+        plane_z = convex_hull.get_average_z()
+        spawn_z = self._occluder_spawn_z + plane_z + -bounding_box[0][2] + self._data_generation_config["spawn_z_offset"]
+        if(spawn_z > self._data_generation_config["max_spawn_z"]):
+            # spawn_z = self._data_generation_config["max_spawn_z"]
+            return False
+        # carb.log_error("Spawning Object: "+ trimesh_obj.name+", z: "+str(spawn_z))
+
+        # get spawning plane bounding box
+        convex_hull_bounds = convex_hull.get_convex_hull()
+        convex_hull_x_bounds = np.asarray(convex_hull_bounds).T[0]
+        convex_hull_y_bounds = np.asarray(convex_hull_bounds).T[1]
+        convex_min_x = min(convex_hull_x_bounds)
+        convex_max_x = max(convex_hull_x_bounds)
+        convex_min_y = min(convex_hull_y_bounds)
+        convex_max_y = max(convex_hull_y_bounds)
+
+        # get object bounding box
+        bb_origin = [(bounding_box[0][0]+bounding_box[1][0])/2, (bounding_box[0][0]+bounding_box[1][0])/2]
+        bb_width = abs(bounding_box[0][0]-bounding_box[1][0])
+        bb_height = abs(bounding_box[0][1]-bounding_box[1][1])
+        # carb.log_error("origin: "+str(bb_origin)+", width: "+str(bb_width)+", height: "+str(bb_height))
+
+        for i in range(self._data_generation_config["num_attemp_times"]):
+            random_origin_x = random.uniform(convex_min_x, convex_max_x)
+            random_origin_y = random.uniform(convex_min_y, convex_max_y)
+            bb_origin[0] += random_origin_x
+            bb_origin[1] += random_origin_y
+            rot_z = random.uniform(-np.pi,np.pi)
+            bb = BOUNDING_BOX_2D(bb_origin, bb_width, bb_height, rot_z)
+            if(convex_hull.are_points_inside_convex_hull(bb._get_bb_vertices())):
+                # carb.log_error("i: "+str(i)+", bb: "+str(bb._get_bb_vertices())+", convex_hull: "+str(convex_hull_bounds[:3]) )
+                place_x = bb_origin[0]
+                place_y = bb_origin[1]
+
+                # Spawn Object
+                object = rep.create.from_usd(trimesh_obj.usd_path, semantics=[("class", trimesh_obj.name)])
+                with object:
+                    rep.modify.pose(position=(place_x, place_y, spawn_z), rotation_x = -180, rotation_z = -90, scale=self._data_generation_config["model_scaling"])
+                self._replicator_objects.append(object)
+
+                return True
+        return False
+
+    def _add_occluder_to_sim(self, obj_index):
+        from omni.isaac.core.utils.stage import add_reference_to_stage
+        from omni.isaac.core.prims import XFormPrim
+        from omni.physx.scripts import utils
+        from pxr import UsdPhysics
+
+        # Load Object
+        trimesh_occluder = self._trimesh_occluders[obj_index]
+        add_reference_to_stage(usd_path=trimesh_occluder.usd_path, prim_path="/World/"+trimesh_occluder.name)
+
+        # Make it a rigid body
+        occluder_prim = self._stage.GetPrimAtPath("/World/"+trimesh_occluder.name)
+        utils.setRigidBody(occluder_prim, "convexDecomposition", False)
+        mass_api = UsdPhysics.MassAPI.Apply(occluder_prim)
+
+        # Get Bounding Box
+        bounding_box = trimesh_occluder.bb
+
+        self._occluder_top_z =abs(bounding_box[1][1]-bounding_box[0][1])
+        self._occluder_x_lb = bounding_box[0][0]
+        self._occluder_x_ub = bounding_box[1][0]
+        self._occluder_y_lb = bounding_box[0][2]
+        self._occluder_y_ub = bounding_box[1][2]
+
+        # Add to Sim
+        spawn_z = abs(bounding_box[0][1]) + 0.01
+        if(spawn_z > self._data_generation_config["max_spawn_z"]):
+            spawn_z = self._data_generation_config["max_spawn_z"]
+        self._occluder_spawn_z = spawn_z
+        carb.log_warn("Spawning Occluder: "+ trimesh_occluder.name+", z: "+str(spawn_z))
+        self._world.scene.add(XFormPrim(prim_path="/World/"+trimesh_occluder.name, 
+                                        name=trimesh_occluder.name, 
+                                        position=[self._occluder_spawn_x, self._occluder_spawn_y, self._occluder_spawn_z], 
+                                        orientation=[0.7071068, 0.7071068, 0, 0], 
+                                        scale=self._data_generation_config["model_scaling"]))
+
+    def _read_objects(self, dataset_path):
+        for root, dirs, files in os.walk(dataset_path, topdown=False):
+            for name in dirs:
+                path = os.path.join(root, name)
+                obj_label=self._extract_model_type(path)
+                obj_name=self._extract_model_name(path)
+                if(os.path.join(root, name)[-14:] == "Scan_converted"):
+                    # self._objs[path+'/Simp_obj.usd'] = [obj_label, obj_name]
+                    self._trimesh_objects.append(Object(obj_path=path[:-10]+'/Simp.obj', 
+                                                        usd_path = path+'/Simp_obj.usd',
+                                                        scale=self._data_generation_config["model_scaling"],
+                                                        eular_angles=[1.57, 0, 0],
+                                                        label=obj_label, 
+                                                        name=obj_name))
+                    
+    def _read_occluders(self, dataset_path):
+        for root, dirs, files in os.walk(dataset_path, topdown=False):
+            for name in dirs:
+                path = os.path.join(root, name)
+                occluder_label=self._extract_model_type(path)
+                occluder_name=self._extract_model_name(path)
+                if(os.path.join(root, name)[-14:] == "Scan_converted"):
+                    self._trimesh_occluders.append(Object(obj_path=path[:-10]+'/Simp.obj', 
+                                                        usd_path = path+'/Simp_obj.usd',
+                                                        scale=self._data_generation_config["model_scaling"],
+                                                        eular_angles=[1.57, 0, 0],
+                                                        label=occluder_label, 
+                                                        name=occluder_name))
+
+    def _spawn_objects(self):
+        from omni.physx.scripts import utils
+        from pxr import UsdPhysics
+        import omni.replicator.core as rep
+
+        num_spawned_objects = 0
+        usd_spawned = []
+        while(num_spawned_objects < self._data_generation_config["object_num"]):
+            random_index = random.randint(0, len(self._trimesh_objects)-1)
+            if(self._trimesh_objects[random_index].usd_path in usd_spawned):
+                pass
+            else:
+                if(self._simulation_app.is_running()):
+                    self._trimesh_objects[random_index]._load_mesh()
+                    if(self._add_object_to_sim(random_index)):
+                        usd_spawned.append(self._trimesh_objects[random_index].usd_path)
+                        num_spawned_objects += 1
+                        self._selected_object_indices.append(random_index)
+        
+        # Make objects all rigid bodies
+        for object in self._replicator_objects:
+            prim_path = rep.utils.get_node_targets(object.node, "inputs:primsIn")[0]
+            object_prim = self._stage.GetPrimAtPath(prim_path)
+            utils.setRigidBody(object_prim, "triangleMesh", False)
+            mass_api = UsdPhysics.MassAPI.Apply(object_prim)
+            self._step(10)
+        self._step(100)
+
+    def _spawn_occluder(self):
+        random_index = random.randint(0, len(self._trimesh_occluders)-1)
+        self._selected_occluder_index = random_index
+        if(self._simulation_app.is_running()):
+            self._trimesh_occluders[random_index]._load_mesh()
+            carb.log_error(self._trimesh_occluders[random_index]._mesh)
+            self._trimesh_occluders[random_index]._find_placable_areas()
+            self._add_occluder_to_sim(random_index)
+            self._step(10)
+
+    def _step(self, step_times, render=True, step_sim=True):
+        for i in range(step_times):
+            if(self._simulation_app.is_running()):
+                self._world.step(render=render, step_sim=step_sim)
+
+    def _export_usd_scene(self, scene_path, scene_id=0):
+        self._stage.GetRootLayer().Export(scene_path+"/scene_"+str(scene_id)+"/scene.usd")
+
+    def _generate_dataset(self):
+        c = 0
+        for r in range(self._num_r):
+            for h in range(self._num_h):
+                for v in range(self._num_v):
+                    sampling_radius = self._data_generation_config["min_radius"] + r*self._data_generation_config["dr"]
+                    h_angle_in_rad = (self._data_generation_config["start_h_angle"] + self._data_generation_config["dhfov"] * h) / 180 * math.pi
+                    v_angle_in_rad = (0 + self._data_generation_config["dvfov"] * v) / 180 * math.pi
+                    look_at_position = (self._data_generation_config["look_at_position"][0]+((random.random()-0.5)*self._data_generation_config["look_at_noise"]),
+                                        self._data_generation_config["look_at_position"][1]+((random.random()-0.5)*self._data_generation_config["look_at_noise"]),
+                                        self._data_generation_config["look_at_position"][2]+((random.random()-0.5)*self._data_generation_config["look_at_noise"]),)
+                    # carb.log_warn("r: "+str(sampling_radius)+", h: "+str(h_angle_in_rad)+", v: "+str(v_angle_in_rad))
+
+                    px = self._data_generation_config["look_at_position"][0] + sampling_radius*math.sin(v_angle_in_rad)*math.cos(h_angle_in_rad)
+                    py = self._data_generation_config["look_at_position"][1] + sampling_radius*math.cos(v_angle_in_rad)*math.cos(h_angle_in_rad)
+                    pz = self._data_generation_config["look_at_position"][2] + sampling_radius*math.sin(h_angle_in_rad)
+                    dx = look_at_position[0] - px
+                    dy = look_at_position[1] - py
+                    dz = look_at_position[2] - pz
+                    roll = (random.random()-0.5)*30
+                    pitch = math.atan2(dz, math.sqrt(dx**2+dy**2))/math.pi*180
+                    yaw = math.atan2(-dy, -dx)/math.pi*180
+
+                    _camera_prim = self._stage.GetPrimAtPath("/Replicator/DriverCam_Xform")
+                    _camera_prim.GetAttribute("xformOp:translate").Set((px, py, pz))
+                    _camera_prim.GetAttribute("xformOp:rotateXYZ").Set((roll, pitch, yaw))
+                    carb.log_warn("frame: "+str(c)+"x: "+str(px)+", y: "+str(py)+", z: "+str(pz)+", roll: "+str(roll)+", pitch: "+str(pitch)+", yaw: "+str(yaw))
+                    self._rep.orchestrator.step()
+                    self._step(1)
+                    c+=1    
+
+    def _is_scene_valid(self):
+        trimesh_occluder = self._trimesh_occluders[self._selected_occluder_index]
+        occluder_prim_path = "/World/"+trimesh_occluder.name
+        occluder_dc_object = self._dc.get_rigid_body(occluder_prim_path)
+        occluder_pose = self._dc.get_rigid_body_pose(occluder_dc_object)
+
+        occluder_actual_position = occluder_pose.p
+        occluder_reference_position = [self._occluder_spawn_x, self._occluder_spawn_y, self._occluder_spawn_z]
+        # carb.log_error("current occluder pose x: "+str(occluder_pose.p[0])+
+        #                ", current occluder pose y: "+str(occluder_pose.p[1])+
+        #                ", current occluder pose z: "+str(occluder_pose.p[2]))
+        # carb.log_error("reference occluder pose x: "+str(self._occluder_spawn_x)+
+        #                ", reference occluder pose y: "+str(self._occluder_spawn_y)+
+        #                ", reference occluder pose z: "+str(self._occluder_spawn_z))
+        distance = distance_3d(occluder_actual_position, occluder_reference_position)
+        carb.log_warn("distance between occluder's reference and actual positions: "+str(distance))
+        if(distance >= self._data_generation_config["distance_threshold"]):
+            carb.log_error("Occluder is moved too much after spawning objects, scene generation failed!")
+            return False
+        return True
+    
+    def _refresh_objects_and_occluders(self):
+        self._occluder_top_z = 0
+        self._occluder_spawn_x = 0
+        self._occluder_spawn_y = 0
+        self._occluder_spawn_z = 0
+        self._occluder_x_lb = 0
+        self._occluder_x_ub = 0
+        self._occluder_y_lb = 0
+        self._occluder_y_ub = 0
+
+        self._selected_occluder_index = 0
+        self._selected_object_indices = []
+
+        self._replicator_objects = []
+
+    def _main(self):
+        self._add_background(self._data_generation_config["background_path"])
+        self._read_objects(self._data_generation_config["object_dataset_path"])
+        self._read_occluders(self._data_generation_config["occluder_dataset_path"])
+        # c = 0
+        # while(c < self._data_generation_config["num_scenes"]):
+        #     self._spawn_occluder()
+        #     self._spawn_objects()
+        #     if(self._is_scene_valid()):
+        #         self._setup_replicator(scene_id=c)
+        #         self._generate_dataset()
+        #         self._export_usd_scene(self._data_generation_config["data_save_path"], scene_id=c)
+        #     self._refresh_objects_and_occluders()
+
+        self._spawn_occluder()
+        self._spawn_objects()
+        self._setup_replicator(scene_id=0)
+        self._generate_dataset()
+        self._export_usd_scene(self._data_generation_config["data_save_path"], scene_id=0)
+        self._step(10000)
+        self._simulation_app.close()
+
+
+abba = GENERATE_BOJECTS()

data_generation/src/generate_objects_utils.py

@@ -0,0 +1,290 @@
+import numpy as np
+import trimesh
+from sklearn import linear_model
+import copy
+import math
+
+def distance_3d(a, b):
+    return np.sqrt((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2)
+
+class CONVEX_HULL:
+    def __init__(self, points) -> None:
+        self.average_z = 0
+        self.highest_z = -np.inf
+        self.convex_hull = self.polygon_scan(points)
+        self.calc_average_z()
+        self.calc_highest_z()
+
+    def distance(self, a, b):
+        return np.sqrt((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2)
+
+    def cross_product(self, a, b, c):
+        return (b[1]-a[1])*(c[0]-b[0]) - (b[0]-a[0])*(c[1]-b[1])
+
+    def dot_product(self, a, b, c):
+        ab = [b[0] - a[0], b[1] - a[1]]
+        bc = [c[0] - b[0], c[1] - b[1]]
+        return ab[0] * bc[0] + ab[1] * bc[1]
+
+    def angle(self, a, b, c):
+        cp = self.cross_product(a, b, c)
+        dp= self.dot_product(a, b, c)
+        angle_radians = math.atan2(cp, dp)
+        angle_degrees = math.degrees(angle_radians)
+
+        return angle_degrees
+
+    def new_point_orientation(self, pivot, p1, p2):
+        orientation = self.cross_product(pivot, p1, p2)
+        if orientation == 0:
+            return 0
+        elif(orientation > 0):
+            return 1
+        else:
+            return -1
+
+    def polygon_scan(self, points_in):
+        points = copy.deepcopy(points_in)
+        n = len(points)
+        if n < 3:
+            print("Convec Hull required at least 3 points.")
+            return
+        else:
+            if(n==3 and self.new_point_orientation[points[0],points[1],points[2]]==0):
+                print("3 points are in one line.")
+                return
+            else:
+                pivot_index, pivot_point = min(enumerate(points), key=lambda point: point[1]) # 优先根据y值排序，若y相同，则找到相同y下x值最小的点
+                start_point = copy.deepcopy(pivot_point)
+                start_point[1] -= 1
+                convex_hull = [start_point, pivot_point]
+                closed = False
+
+                while not closed:
+                    max_x = -np.inf
+                    max_x_idx = 0
+                    for p_idx in range(len(points)):
+                        point = points[p_idx]
+                        x = self.angle(convex_hull[-2], convex_hull[-1], point)
+                        if(x > max_x and x != 180 and x != 0):
+                            max_x = x
+                            max_x_idx = p_idx
+                    if(len(points) == 0):
+                        closed = True
+                    else:
+                        new_point = points[max_x_idx]
+                        if(new_point[0] == pivot_point[0] and new_point[1] == pivot_point[1]):
+                            closed = True
+                        else:
+                            convex_hull.append(new_point)
+                            points.pop(max_x_idx)
+
+                convex_hull.pop(0)
+                return convex_hull
+
+    def is_point_inside_convex_hull(self, point_in):
+        point = copy.deepcopy(point_in)
+        new_convex_hull_points = copy.deepcopy(self.convex_hull)
+        new_convex_hull_points.append(point)
+        new_convex_hull = self.polygon_scan(new_convex_hull_points)
+        ans = (new_convex_hull == self.convex_hull)
+        return ans
+
+    def are_points_inside_convex_hull(self, points_in):
+        points = copy.deepcopy(points_in)
+        ans = True
+        for point in points:
+            p_ans =  self.is_point_inside_convex_hull(point)
+            ans = ans * p_ans
+        return ans
+
+    def get_convex_hull(self):
+        return self.convex_hull
+    
+    def calc_average_z(self):
+        for p in self.convex_hull:
+            self.average_z += p[2]/len(self.convex_hull)
+
+    def get_average_z(self):
+        return self.average_z
+
+    def calc_highest_z(self):
+        for p in self.convex_hull:
+            if(p[2]>self.highest_z):
+                self.highest_z = p[2]
+
+    def get_highest_z(self):
+        return self.highest_z
+
+    
+
+class Object:
+    def __init__(self, obj_path, usd_path=None, scale=[1, 1, 1], eular_angles=[0, 0, 0], max_num_planes=2,
+                 label=None, name=None, ) -> None:
+        # trimesh
+        self.obj_path = obj_path
+        self.usd_path = usd_path
+        self.scale = scale
+        self.eular_angles = eular_angles
+        self.label = label
+        self.name = name
+        self.bb = None
+
+        # get placable points
+        self.placable_points = []
+        
+        # ransac
+        self.max_num_planes = max_num_planes
+        self.X = []
+        self.y = []
+        self.plane_point_groups = []
+        self.plane_coef_groups = []
+
+        # convex_hull
+        self.convex_hulls = []
+        self.convex_hulls_z = []
+        
+    def _load_mesh(self):
+        # Load mesh
+        self._mesh = trimesh.load(self.obj_path)
+        self._mesh.apply_scale(self.scale)
+
+        # Get Bounding Box
+        self.bb = self._mesh.bounds
+
+        # Rotate Mesh
+        self._rotate_mesh(self.eular_angles)
+
+    def _find_placable_areas(self):
+        # Find placable points
+        self._get_placable_points()
+        self._convert_placable_points_to_sklearn_X_y()
+        self._get_plane_point_groups()
+        self._get_convex_hulls()
+
+    def _rotate_mesh(self, rot_rpy):
+        center = [0, 0, 0]
+        angle = rot_rpy[0]
+        direction = [1, 0, 0]
+        rot_matrix_r = trimesh.transformations.rotation_matrix(angle, direction, center)
+        self._mesh.apply_transform(rot_matrix_r)
+        angle = rot_rpy[1]
+        direction = [0, 1, 0]
+        rot_matrix_p = trimesh.transformations.rotation_matrix(angle, direction, center)
+        self._mesh.apply_transform(rot_matrix_p)
+        angle = rot_rpy[2]
+        direction = [0, 0, 1]
+        rot_matrix_y = trimesh.transformations.rotation_matrix(angle, direction, center)
+        self._mesh.apply_transform(rot_matrix_y)
+
+    def _get_placable_points(self):
+        gravity=np.array([0, 0, -1.0])
+        support_facet_indices = np.argsort(self._mesh.facets_area)
+        for idx in support_facet_indices:
+            if(np.isclose(self._mesh.facets_normal[idx].dot(gravity), 1.0, atol=0.15)):
+                facet_boundary = self._mesh.facets_boundary[idx]
+                for bondary_vertices_indices in facet_boundary:
+                    for boundary_vertex_index in bondary_vertices_indices:
+                        point = self._mesh.vertices[boundary_vertex_index]
+                        self.placable_points.append(list(point))
+        return self.placable_points
+    
+    def _convert_placable_points_to_sklearn_X_y(self):
+        for p in self.placable_points:
+            self.X.append([p[0], p[1]])
+            self.y.append(p[2])
+        self.X = np.array(self.X)
+        self.y = np.array(self.y)
+        return self.X, self.y
+    
+    def _get_plane_point_groups(self):
+        ransac = linear_model.RANSACRegressor()
+        num_planes = 0
+        while(num_planes < self.max_num_planes):
+            plane_points = []
+            plane_coefs = []
+            num_popped_element = 0
+            if(len(self.X) <= 2):
+                break
+            else:
+                ransac.fit(self.X, self.y)
+                inlier_mask = ransac.inlier_mask_ # a list of bool values indicating whether the element is inlier point
+                plane_coefficients = ransac.estimator_.coef_
+                plane_intercept = ransac.estimator_.intercept_
+                plane_coefs = [plane_coefficients[0], plane_coefficients[1], plane_intercept]
+                for i in range(len(self.X)):
+                    if(len(self.X) > 0 and inlier_mask[i] == True):
+                        X = self.X[i - num_popped_element]
+                        y = self.y[i - num_popped_element]
+                        plane_points.append([X[0], X[1], y])
+                        self.X = np.delete(self.X, i-num_popped_element, 0)
+                        self.y = np.delete(self.y, i-num_popped_element, 0)
+                        num_popped_element+=1
+
+                self.plane_point_groups.append(plane_points)
+                self.plane_coef_groups.append(plane_coefs)
+                num_planes += 1
+        return self.plane_point_groups, self.plane_coef_groups
+    
+    def _get_convex_hulls(self):
+        for plane_points in self.plane_point_groups:
+            ch = CONVEX_HULL(plane_points)
+            self.convex_hulls.append(ch)
+
+class BOUNDING_BOX_2D:
+    def __init__(self, origin, width, height, rot_z=0) -> None:
+        self.origin = origin
+        self.origin_x = origin[0]
+        self.origin_y = origin[1]
+        self.width = width
+        self.height = height
+        self.rot_z = rot_z
+
+        self.setup()
+
+    def setup(self):
+        self._get_bb_vertices()
+        self._apply_transform_to_bb_vertices()
+
+    def _get_bb_vertices(self):
+        self.bb_vertics = [[self.origin_x-self.width/2, self.origin_y-self.height/2],
+                           [self.origin_x-self.width/2, self.origin_y+self.height/2],
+                           [self.origin_x+self.width/2, self.origin_y-self.height/2],
+                           [self.origin_x+self.width/2, self.origin_y+self.height/2],]
+        return self.bb_vertics
+
+    def _apply_transform_to_bb_vertices(self):
+        origin = np.array([[1, 0, 0, self.origin_x],
+                           [0, 1, 0, self.origin_y],
+                           [0, 0, 1, 0],
+                           [0, 0, 0, 1]])
+        T = np.array([[np.cos(self.rot_z), -np.sin(self.rot_z), 0, 0],
+                      [np.sin(self.rot_z), np.cos(self.rot_z),  0, 0],
+                      [0,                  0,                   1, 0],
+                      [0,                  0,                   0, 1]])
+        T_left_up = np.array([[1, 0, 0, -self.width/2],
+                              [0, 1, 0, self.height/2],
+                              [0, 0, 1, 0],
+                              [0, 0, 0, 1]])
+        T_left_down = np.array([[1, 0, 0, -self.width/2],
+                                [0, 1, 0, -self.height/2],
+                                [0, 0, 1, 0],
+                                [0, 0, 0, 1]])
+        T_right_up = np.array([[1, 0, 0, self.width/2],
+                               [0, 1, 0, self.height/2],
+                               [0, 0, 1, 0],
+                               [0, 0, 0, 1]])
+        T_right_down = np.array([[1, 0, 0, self.width/2],
+                               [0, 1, 0, -self.height/2],
+                               [0, 0, 1, 0],
+                               [0, 0, 0, 1]])
+        transformed_origin = origin.dot(T)
+        tv0 = transformed_origin.dot(T_left_down)
+        tv1 = transformed_origin.dot(T_left_up)
+        tv2 = transformed_origin.dot(T_right_down)
+        tv3 = transformed_origin.dot(T_right_up)
+        transformed_vertices = [tv0, tv1, tv2, tv3]
+    
+        for vertex_idx in range(len(self.bb_vertics)):
+            self.bb_vertics[vertex_idx][0] = transformed_vertices[vertex_idx][0][3]
+            self.bb_vertics[vertex_idx][1] = transformed_vertices[vertex_idx][1][3]

data_generation/tools/convert_dataset.py

@@ -0,0 +1,83 @@
+import argparse
+import asyncio
+import os
+
+import omni
+from omni.isaac.kit import SimulationApp
+
+
+async def convert(in_file, out_file, load_materials=True):
+    # This import causes conflicts when global
+    import omni.kit.asset_converter
+
+    def progress_callback(progress, total_steps):
+        pass
+
+    converter_context = omni.kit.asset_converter.AssetConverterContext()
+    # setup converter and flags
+    converter_context.ignore_materials = not load_materials
+    # converter_context.ignore_animation = False
+    # converter_context.ignore_cameras = True
+    # converter_context.single_mesh = True
+    # converter_context.smooth_normals = True
+    # converter_context.preview_surface = False
+    # converter_context.support_point_instancer = False
+    # converter_context.embed_mdl_in_usd = False
+    # converter_context.use_meter_as_world_unit = True
+    # converter_context.create_world_as_default_root_prim = False
+    instance = omni.kit.asset_converter.get_instance()
+    task = instance.create_converter_task(in_file, out_file, progress_callback, converter_context)
+    success = True
+    while True:
+        success = await task.wait_until_finished()
+        if not success:
+            await asyncio.sleep(0.1)
+        else:
+            break
+    return success
+
+
+def asset_convert(folders):
+    supported_file_formats = ["stl", "obj", "fbx"]
+    for folder in folders:
+        local_asset_output = folder + "_converted"
+        result = omni.client.create_folder(f"{local_asset_output}")
+
+    for folder in folders:
+        print(f"\nConverting folder {folder}...")
+
+        (result, models) = omni.client.list(folder)
+        for i, entry in enumerate(models):
+            model = str(entry.relative_path)
+            model_name = os.path.splitext(model)[0]
+            model_format = (os.path.splitext(model)[1])[1:]
+            # Supported input file formats
+            if model_format in supported_file_formats:
+                input_model_path = folder + "/" + model
+                converted_model_path = folder + "_converted/" + model_name + "_" + model_format + ".usd"
+                if not os.path.exists(converted_model_path):
+                    status = asyncio.get_event_loop().run_until_complete(
+                        convert(input_model_path, converted_model_path, True)
+                    )
+                    if not status:
+                        print(f"ERROR Status is {status}")
+                    print(f"---Added {converted_model_path}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Process some integers.')
+    parser.add_argument('dataset_path', type=str, help='path to the dataset')
+    args = parser.parse_args()
+
+    folders = []
+    for root, dirs, files in os.walk(args.dataset_path, topdown=False):
+        for name in dirs:
+            path = os.path.join(root, name)
+            if(os.path.join(root, name)[-4:] == "Scan"):
+                folders.append(path)
+
+    kit = SimulationApp()
+    from omni.isaac.core.utils.extensions import enable_extension
+    enable_extension("omni.kit.asset_converter")
+    asset_convert(folders)
+    kit.close()

data_generation/tools/get_view.py

@@ -0,0 +1,240 @@
+import os
+import pickle
+import pybullet as p
+import time
+import pybullet_data
+import numpy as np
+import matplotlib.pyplot as plt
+from PIL import Image 
+import open3d as o3d
+import cv2
+import math
+import json
+
+
+class GenerateScene:
+    def __init__(self, dataset_path, scene_path, output_path, camera_params) -> None:
+        self._init_variables()
+        self._load_object_dataset(dataset_path)
+        self._load_scene(scene_path)
+        self._set_output_path(output_path)
+        self._set_camera_params(camera_params)
+
+
+    def _init_variables(self):
+        self.object_paths = {}
+        self.scene = {}
+        self.object_model_scale = [0.001, 0.001, 0.001]
+        self.output_path = None
+        self.camera_params = None
+        self.segmentation_labels = {}
+
+
+    def _extract_model_name(self, path):
+        # Specify it according to specific file directory
+        NUM_SLASH_BEFORE_NAME = 1
+
+        num_slash = 0
+        object_name_str = []
+
+        for i in range(len(path)):
+            index = len(path) -1 - i
+            char = path[index]
+
+            if(num_slash == NUM_SLASH_BEFORE_NAME):
+                object_name_str.append(char)
+
+            if(char == "/"):
+                num_slash += 1
+
+        object_name_str.reverse()
+        object_name_str = ''.join(object_name_str[1:])
+        return object_name_str
+
+
+    def _load_object_dataset(self, dataset_path):
+        if(dataset_path[-1] == "/"):
+            dataset_path = dataset_path[:-1]
+        for root, dirs, files in os.walk(dataset_path, topdown=False):
+            for name in dirs:
+                path = os.path.join(root, name)
+                if(os.path.join(root, name)[-4:] == "Scan"):
+                    name = self._extract_model_name(path)
+                    self.object_paths[name] = path+"/Simp.obj"
+
+
+    def _load_scene(self, scene_path):
+        if(scene_path[-1] == "/"):
+            scene_path = scene_path[:-1]
+        scene_path = scene_path + "/scene.pickle"
+        self.scene = pickle.load(open(scene_path, 'rb'))
+
+    
+    def _set_output_path(self, output_path):
+        self.output_path = output_path
+        if(self.output_path[-1] == "/"):
+            self.output_path = self.output_path[:-1]
+
+    
+    def _set_camera_params(self, camera_params):
+        self.camera_params = camera_params
+
+
+    def load_camera_pose_from_frame(self, camera_params_path):
+        with open(camera_params_path, "r") as f:
+            camera_params = json.load(f)
+        
+        view_transform = camera_params["cameraViewTransform"]
+        print(view_transform)
+        view_transform = np.resize(view_transform, (4,4))
+        view_transform = np.linalg.inv(view_transform).T
+        offset = np.mat([[1,0,0,0],[0,-1,0,0],[0,0,-1,0],[0,0,0,1]])
+        view_transform = view_transform.dot(offset)
+        print(view_transform)
+        return view_transform
+
+
+    def _load_obj_to_pybullet(self, obj_file_path, position, orientation, scale):
+        visual_ind = p.createVisualShape(
+            shapeType=p.GEOM_MESH,
+            fileName=obj_file_path,
+            rgbaColor=[1, 1, 1, 1],
+            specularColor=[0.4, 0.4, 0],
+            visualFramePosition=[0, 0, 0],
+            meshScale=scale)
+        p.createMultiBody(
+            baseMass=1,
+            baseVisualShapeIndex=visual_ind,
+            basePosition=position,
+            baseOrientation=orientation,
+            useMaximalCoordinates=True)
+    
+
+    def _render_image(self, camera_pose):
+        width = self.camera_params["width"]
+        height = self.camera_params["height"]
+        fov = self.camera_params["fov"]
+        aspect = width / height
+        near = self.camera_params["near"]
+        far = self.camera_params["far"]
+
+        T = np.mat([[1,0,0,0],
+                    [0,1,0,0],
+                    [0,0,1,1],
+                    [0,0,0,1]])
+        look_at_T = camera_pose.dot(T)
+        view_matrix = p.computeViewMatrix([camera_pose[0,3], camera_pose[1,3], camera_pose[2,3]], 
+                                          [look_at_T[0,3], look_at_T[1,3], look_at_T[2,3]], 
+                                          [-camera_pose[0,1], -camera_pose[1,1], -camera_pose[2,1]])
+        projection_matrix = p.computeProjectionMatrixFOV(fov, aspect, near, far)
+
+        # Get depth values using the OpenGL renderer
+        images = p.getCameraImage(width, height, view_matrix, projection_matrix, renderer=p.ER_BULLET_HARDWARE_OPENGL)
+        rgb = images[2]
+        depth = images[3]
+        seg = images[4]
+
+        rgb_image = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB)
+        cv2.imwrite(self.output_path+'/rgb.jpg',rgb_image)
+
+        depth_image = far * near / (far - (far - near) * depth)
+        depth_image = np.asanyarray(depth_image).astype(np.float32) * 1000.0 
+        depth_image_array = depth_image
+        depth_image = (depth_image.astype(np.uint16))
+        depth_image = Image.fromarray(depth_image)
+        depth_image.save(self.output_path+'/depth.png')
+
+        cv2.imwrite(self.output_path+'/seg.jpg', seg)
+
+        id = 0
+        for object_name in self.scene.keys():
+            self.segmentation_labels[str(id+1)] = object_name
+            id += 1
+        with open(self.output_path+"/seg_labels.json", 'w') as seg_labels:
+            json.dump(self.segmentation_labels, seg_labels)
+
+        with open(self.output_path+"/cam_intrinsics.json", 'w') as cam_intrinsics:
+            json.dump(self.camera_params, cam_intrinsics)
+        
+
+    def generate_images(self, camera_pose):
+        physicsClient = p.connect(p.GUI)
+        p.setAdditionalSearchPath(pybullet_data.getDataPath())
+        p.setGravity(0,0,0)
+        p.loadURDF("plane100.urdf")
+
+        for obj_name in self.scene.keys():
+            orientation = self.scene[obj_name]["rotation"]
+            position = self.scene[obj_name]["position"]
+            self._load_obj_to_pybullet(obj_file_path=self.object_paths[obj_name], 
+                                    position=position, 
+                                    orientation=orientation, 
+                                    scale=self.object_model_scale)
+        self._render_image(camera_pose)
+        p.stepSimulation()
+        p.disconnect()
+
+
+    def visualize_pcd(self):
+        color_image = o3d.io.read_image(self.output_path+'/rgb.jpg')
+        depth_image = o3d.io.read_image(self.output_path+'/depth.png')
+
+        rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(color_image, depth_image,convert_rgb_to_intensity=False)
+        intrinsic = o3d.camera.PinholeCameraIntrinsic(
+            o3d.camera.PinholeCameraIntrinsicParameters.Kinect2DepthCameraDefault )
+        intrinsic.set_intrinsics(width=self.camera_params["width"], 
+                                 height=self.camera_params["height"],
+                                 fx=self.camera_params["fx"], 
+                                 fy=self.camera_params["fy"], 
+                                 cx=self.camera_params["cx"], 
+                                 cy=self.camera_params["cy"])
+
+        point_cloud = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, intrinsic)
+        o3d.visualization.draw_geometries([point_cloud])
+
+    def print_scene(self):
+        print("================= Scene Objects: =================")
+        print(self.scene.keys())
+        print("==================================================")
+
+
+DATASET_PATH = "/home/hzx/Downloads/OmniObject3d-simplified/output"
+SCENE_PATH = "/home/hzx/Projects/ActivePerception/data_generation/output/scene_0/"
+OUTPUT_PATH = "/home/hzx/Projects/ActivePerception/data_generation/output/"
+FRAME_PATH = "/home/hzx/Projects/ActivePerception/data_generation/output/scene_0/camera_params_0119.json"
+
+ISAAC_SIM_CAM_H_APERTURE = 20.955 # Isaac Sim里读取的
+ISAAC_SIM_CAM_V_APERTURE = 15.2908 # Isaac Sim里读取的
+ISAAC_SIM_FOCAL_LENGTH = 39 # 试出来的，其实Isaac Sim里原本是24
+ISAAC_SIM_CAM_D_APERTURE = math.sqrt(ISAAC_SIM_CAM_H_APERTURE**2 + ISAAC_SIM_CAM_V_APERTURE**2)
+
+
+CAM_WIDTH = 640
+CAM_HEIGHT = 480
+CAM_FOV = 2*math.atan(ISAAC_SIM_CAM_D_APERTURE/(2*ISAAC_SIM_FOCAL_LENGTH))/math.pi*180
+CAM_NEAR = 0.001 # 成像最近距离
+CAM_FAR = 10 # 成像最远距离
+CAM_CX = CAM_WIDTH/2
+CAM_CY = CAM_HEIGHT/2
+CAM_FX = 1/math.tan(CAM_FOV*math.pi/180.0/2.0)*CAM_WIDTH/2
+CAM_FY = 1/(CAM_HEIGHT/CAM_WIDTH*math.tan(CAM_FOV*math.pi/180.0/2.0))*CAM_HEIGHT/2
+
+CAMERA_PARAMS = {
+    "width": CAM_WIDTH,
+    "height": CAM_HEIGHT,
+    "fov": CAM_FOV,
+    "near": CAM_NEAR,
+    "far": CAM_FAR,
+    "cx": CAM_CX,
+    "cy": CAM_CY,
+    "fx": CAM_FX,
+    "fy": CAM_FY,
+}
+
+
+if __name__ == "__main__":
+    gs = GenerateScene(DATASET_PATH, SCENE_PATH, OUTPUT_PATH, CAMERA_PARAMS)
+    gs.print_scene()
+    cam_pose = gs.load_camera_pose_from_frame(FRAME_PATH)
+    gs.generate_images(cam_pose) # 在OUTPUT_PATH路径下生成rgb、depth、segmentation图
+    # gs.visualize_pcd()