Sensors
Sensors are important for collecting information about surroundings. By default, all environments provide 3 basic sensors:
Lidar
SideDetector
LaneLineDetector
which are used for detecting moving objects, sidewalks/solid lines, and broken/solid lines respectively.
As these sensors are built based on ray test and don’t need graphics support, they can be used in all modes.
Also, you don’t need to recreate them again, as they are not binded with any objects until perceive() is called and the target object is specified. After collecting results, those ray-based sensors are detached and ready for next use.
You can access them at anywhere through the engine.get_sensor(sensor_id):
from metadrive.envs.base_env import BaseEnv
env = BaseEnv(dict(log_level=50))
env.reset()
lidar = env.engine.get_sensor("lidar")
side_lidar = env.engine.get_sensor("side_detector")
lane_line_lidar = env.engine.get_sensor("lane_line_detector")
print("Available sensors are:", env.engine.sensors.keys())
env.close()
Available sensors are: dict_keys(['lidar', 'side_detector', 'lane_line_detector'])
Add New Sensor
To add new sensors, you should request them by using env_config.
If an sensor is defined as follows:
class MySensor(BaseSensor):
def __init__(self, args_1, args_2, engine)
Then we can create it by:
env_cfg = dict(sensors=dict(new_sensor=(MySensor, args_1, args_2)))
env = MetaDriveEnv(env_cfg)
The following example shows how to create a RGBCamera whose buffer size are width=32, height=16.
Note: for creating cameras or any sensors requiring rendering, please turn on image_observation.
from metadrive.envs.base_env import BaseEnv
from metadrive.component.sensors.rgb_camera import RGBCamera
import cv2
import os
size = (256, 128) if not os.getenv('TEST_DOC') else (16, 16) # for github CI
env_cfg = dict(log_level=50, # suppress log
image_observation=True,
show_terrain=not os.getenv('TEST_DOC'),
sensors=dict(rgb=[RGBCamera, *size]))
env = BaseEnv(env_cfg)
env.reset()
print("Available sensors are:", env.engine.sensors.keys())
cam = env.engine.get_sensor("rgb")
img = cam.get_rgb_array_cpu()
cv2.imwrite("img.png", img)
env.close()
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[2], line 13
7 env_cfg = dict(log_level=50, # suppress log
8 image_observation=True,
9 show_terrain=not os.getenv('TEST_DOC'),
10 sensors=dict(rgb=[RGBCamera, *size]))
12 env = BaseEnv(env_cfg)
---> 13 env.reset()
14 print("Available sensors are:", env.engine.sensors.keys())
15 cam = env.engine.get_sensor("rgb")
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/envs/base_env.py:523, in BaseEnv.reset(self, seed)
521 log_level = self.config.get("log_level", logging.DEBUG if self.config.get("debug", False) else logging.INFO)
522 set_log_level(log_level)
--> 523 self.lazy_init() # it only works the first time when reset() is called to avoid the error when render
524 self._reset_global_seed(seed)
525 if self.engine is None:
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/envs/base_env.py:415, in BaseEnv.lazy_init(self)
413 if engine_initialized():
414 return
--> 415 initialize_engine(self.config)
416 # engine setup
417 self.setup_engine()
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/engine/engine_utils.py:38, in initialize_engine(env_global_config)
35 cls = BaseEngine
36 if cls.singleton is None:
37 # assert cls.global_config is not None, "Set global config before initialization BaseEngine"
---> 38 cls.singleton = cls(env_global_config)
39 else:
40 raise PermissionError("There should be only one BaseEngine instance in one process")
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/engine/base_engine.py:58, in BaseEngine.__init__(self, global_config)
56 self.id_c = dict()
57 self.try_pull_asset()
---> 58 EngineCore.__init__(self, global_config)
59 Randomizable.__init__(self, self.global_random_seed)
60 self.episode_step = 0
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/engine/core/engine_core.py:313, in EngineCore.__init__(self, global_config)
307 self.pbrpipe = init(
308 msaa_samples=4,
309 # use_hardware_skinning=True,
310 use_330=True
311 )
312 else:
--> 313 self.pbrpipe = init(
314 msaa_samples=16,
315 use_hardware_skinning=True,
316 # use_normal_maps=True,
317 use_330=False
318 )
320 self.sky_box = SkyBox(not self.global_config["show_skybox"])
321 self.sky_box.attach_to_world(self.render, self.physics_world)
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/third_party/simplepbr/__init__.py:350, in init(**kwargs)
320 def init(**kwargs):
321 '''Initialize the PBR render pipeline
322 :param render_node: The node to attach the shader too, defaults to `base.render` if `None`
323 :type render_node: `panda3d.core.NodePath`
(...) 347 :type use_hardware_skinning: bool or None
348 '''
--> 350 return Pipeline(**kwargs)
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/third_party/simplepbr/__init__.py:142, in Pipeline.__init__(self, render_node, window, camera_node, taskmgr, msaa_samples, max_lights, use_normal_maps, use_emission_maps, exposure, enable_fog, use_occlusion_maps, use_330, use_hardware_skinning)
139 self._recompile_pbr()
141 # Tonemapping
--> 142 self._setup_tonemapping()
144 self._shader_ready = True
File ~/checkouts/readthedocs.org/user_builds/metadrive-simulator/envs/latest/lib/python3.11/site-packages/metadrive/third_party/simplepbr/__init__.py:287, in Pipeline._setup_tonemapping(self)
281 post_frag_str = _load_shader_str('tonemap.frag', defines)
282 tonemap_shader = p3d.Shader.make(
283 p3d.Shader.SL_GLSL,
284 vertex=post_vert_str,
285 fragment=post_frag_str,
286 )
--> 287 self.tonemap_quad.set_shader(tonemap_shader)
288 self.tonemap_quad.set_shader_input('tex', scene_tex)
289 self.tonemap_quad.set_shader_input('exposure', self.exposure)
AttributeError: 'NoneType' object has no attribute 'set_shader'
from IPython.display import Image
Image(open("img.png", "rb").read())
The log message shows that not only the rgb is created, but a main_camera is provided automatically, which is also an RGB camera rendering into the pop-up window. It can serve as a sensor as well.
Graphics-based Sensors
We provide the following sensors:
Main Camera
RGB Camera
Depth Camera
Semantic Camera
Instance Camera
Lidar (Cloud Point)
The following example mainly uses the semantic camera, but the same method can be applied to other sensors including the point cloud.
Using semantic camera as observation
from metadrive.envs import MetaDriveEnv
from metadrive.component.sensors.semantic_camera import SemanticCamera
import matplotlib.pyplot as plt
import os
size = (256, 128) if not os.getenv('TEST_DOC') else (16, 16) # for github CI
env = MetaDriveEnv(dict(
log_level=50, # suppress log
image_observation=True,
show_terrain=not os.getenv('TEST_DOC'),
sensors={"sementic_camera": [SemanticCamera, *size]},
vehicle_config={"image_source": "sementic_camera"},
stack_size=3,
))
obs, info = env.reset()
for _ in range(5):
obs, r, d, t, i = env.step((0, 1))
env.close()
print({k: v.shape for k, v in obs.items()}) # Image is in shape (H, W, C, num_stacks)
{'image': (128, 256, 3, 3), 'state': (19,)}
plt.subplot(131)
plt.imshow(obs["image"][:, :, :, 0])
plt.subplot(132)
plt.imshow(obs["image"][:, :, :, 1])
plt.subplot(133)
plt.imshow(obs["image"][:, :, :, 2])
<matplotlib.image.AxesImage at 0x7fe68b88bc40>
Retrieve semantic images
from metadrive.envs import MetaDriveEnv
from metadrive.component.sensors.semantic_camera import SemanticCamera
import cv2
import os
size = (256, 128) if not os.getenv('TEST_DOC') else (16, 16) # for github CI
env = MetaDriveEnv(dict(
log_level=50, # suppress log
image_observation=True,
show_terrain=not os.getenv('TEST_DOC'),
sensors={"sementic_camera": [SemanticCamera, *size]},
vehicle_config={"image_source": "sementic_camera"}
))
env.reset()
print("Available sensors are:", env.engine.sensors.keys())
cam = env.engine.get_sensor("sementic_camera")
img = cam.get_image(env.agent)
cv2.imwrite("semantics.png", img)
env.close()
from IPython.display import Image
Image(open("semantics.png", "rb").read())
Available sensors are: dict_keys(['lidar', 'side_detector', 'lane_line_detector', 'sementic_camera'])
Demo on RGB camera
from metadrive.envs.base_env import BaseEnv
from metadrive.component.sensors.rgb_camera import RGBCamera
import cv2
import os
size = (256, 128) if not os.getenv('TEST_DOC') else (16, 16) # for github CI
env_cfg = dict(log_level=50, # suppress log
image_observation=True,
show_terrain=not os.getenv('TEST_DOC'),
sensors=dict(sementic_camera=[RGBCamera, *size]))
env = BaseEnv(env_cfg)
env.reset()
print("Available sensors are:", env.engine.sensors.keys())
cam = env.engine.get_sensor("sementic_camera")
img = cam.get_rgb_array_cpu()
cv2.imwrite("semantics.png", img)
env.close()
from IPython.display import Image
Image(open("semantics.png", "rb").read())