Skip to content

first commit #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: v1.0
Choose a base branch
from
Open

first commit #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 24 additions & 22 deletions gym_unrealcv/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from gym.envs.registration import register
import logging
import pdb
from gym_unrealcv.envs.utils.misc import load_env_setting
logger = logging.getLogger(__name__)
use_docker = False # True: use nvidia docker False: do not use nvidia-docker
Expand Down Expand Up @@ -59,7 +60,7 @@
'{action}{obs}-v{reset}'.format(env=env, target=target, path=path,
action=action, obs=obs, reset=i),
entry_point='gym_unrealcv.envs:UnrealCvTracking_spline',
kwargs={'setting_file': 'tracking/v0/{env}{target}{path}.json'.format(
kwargs={'setting_file': 'tracking_v0/{env}{target}{path}.json'.format(
env=env, target=target, path=path),
'reset_type': reset,
'action_type': action,
Expand All @@ -78,7 +79,7 @@
register(
id='UnrealTrack-{env}-{action}{obs}-v{reset}'.format(env=env, action=action, obs=obs, reset=i),
entry_point='gym_unrealcv.envs:UnrealCvTracking_random',
kwargs={'setting_file': 'tracking/v0/{env}.json'.format(env=env),
kwargs={'setting_file': 'tracking_v0/{env}.json'.format(env=env),
'reset_type': i,
'action_type': action,
'observation_type': obs,
Expand All @@ -88,7 +89,6 @@
max_episode_steps=500
)


# "AD-VAT: An Asymmetric Dueling mechanism for learning Visual Active Tracking", ICLR 2019
# DuelingRoom is the training environment, others are testing environment.
for env in ['DuelingRoom', 'UrbanCity', 'UrbanRoad', 'Garage', 'SnowForest', 'Forest', 'Garden']:
Expand All @@ -98,7 +98,7 @@
for target in ['Ram', 'Nav', 'NavBase', 'NavShort', 'NavFix', 'Internal', 'PZR', 'Adv']:
name = 'UnrealTrack-{env}{target}-{action}{obs}-v{reset}'.format(
env=env, action=action, obs=obs, target=target, reset=i)
setting_file = 'tracking/1v1/{env}.json'.format(env=env)
setting_file = 'tracking_1v1/{env}.json'.format(env=env)
register(
id=name,
entry_point='gym_unrealcv.envs:UnrealCvTracking_1v1',
Expand All @@ -113,7 +113,6 @@
max_episode_steps=500
)


# "Pose-Assisted Multi-Camera Collaboration for Active Object Tracking", AAAI 2020
for env in ['MCRoom', 'Garden', 'UrbanTree']:
for i in range(7): # reset type
Expand Down Expand Up @@ -180,22 +179,25 @@
max_episode_steps=500
)

# Env for general purpose active object tracking
for env in ['City', 'FlexibleRoom', 'FlexibleRoom2', 'Forest', 'UrbanCity', 'UrbanRoad', 'Garage', 'SnowForest', 'Garden', 'DesertRuins', 'BrassGardens', 'EFGus']:
for env in ['MTMC18v3']:
for i in range(7): # reset type
for action in ['Discrete', 'Continuous']: # action type
for obs in ['Color', 'Depth', 'Rgbd', 'Gray', 'CG', 'Mask', 'Pose']: # observation type
name = 'UnrealTrackGeneral-{env}-{action}{obs}-v{reset}'.format(env=env, action=action, obs=obs, target=target, reset=i)
setting_file = 'tracking/general/{env}.json'.format(env=env)
register(
id=name,
entry_point='gym_unrealcv.envs:UnrealCvTracking_general',
kwargs={'setting_file': setting_file,
'reset_type': i,
'action_type': action,
'observation_type': obs,
'reward_type': 'distance',
'docker': use_docker,
},
max_episode_steps=500
)
for obs in ['Color', 'Depth', 'Rgbd', 'Gray']: # observation type
for nav in ['Random', 'Goal', 'Internal', 'None',
'RandomInterval', 'GoalInterval', 'InternalInterval', 'NoneInterval']:
name = 'Unreal{env}-{action}{obs}{nav}-v{reset}'.format(env=env, action=action, obs=obs, nav=nav, reset=i)
setting_file = 'tracking_multicam/{env}.json'.format(env=env)
register(
id=name,
entry_point='gym_unrealcv.envs:UnrealCvTracking_MTMCEnv_v3',
kwargs={'setting_file': setting_file,
'reset_type': i,
'action_type': action,
'observation_type': obs,
'reward_type': 'distance',
'docker': use_docker,
'nav': nav,
'args': None
},
max_episode_steps=None
)
82 changes: 41 additions & 41 deletions gym_unrealcv/envs/navigation/interaction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,9 @@ def __init__(self, env, cam_id=0, port=9000,
def get_observation(self, cam_id, observation_type, mode='direct'):
if observation_type == 'Color':
self.img_color = state = self.read_image(cam_id, 'lit', mode)
elif observation_type == 'Mask':
self.img_color = state = self.read_image(cam_id, 'object_mask', mode)
elif observation_type == 'Depth':
self.img_depth = state = self.read_depth(cam_id)
elif observation_type == 'Rgbd':
elif observation_type == 'Rgbd':
self.img_color = self.read_image(cam_id, 'lit', mode)
self.img_depth = self.read_depth(cam_id)
state = np.append(self.img_color, self.img_depth, axis=2)
Expand All @@ -42,30 +40,28 @@ def get_observation(self, cam_id, observation_type, mode='direct'):
return state

def define_observation(self, cam_id, observation_type, mode='direct'):
state = self.get_observation(cam_id, observation_type, mode)
if observation_type == 'Color' or observation_type == 'CG':
if self.use_gym_10_api:
observation_space = spaces.Box(low=0, high=255, shape=state.shape, dtype=np.uint8) # for gym>=0.10
else:
observation_space = spaces.Box(low=0, high=255, shape=state.shape)

if observation_type == 'Pose' or cam_id < 0:
observation_space = spaces.Box(low=-100, high=100, shape=(6,), dtype=np.float16) # TODO check the range and shape
else:
state = self.get_observation(cam_id, observation_type, mode)
if observation_type == 'Color' or observation_type == 'CG' or observation_type == 'Mask':
if self.use_gym_10_api:
observation_space = spaces.Box(low=0, high=255, shape=state.shape, dtype=np.uint8) # for gym>=0.10
else:
observation_space = spaces.Box(low=0, high=255, shape=state.shape)
elif observation_type == 'Depth':
if self.use_gym_10_api:
observation_space = spaces.Box(low=0, high=100, shape=state.shape, dtype=np.float16) # for gym>=0.10
else:
observation_space = spaces.Box(low=0, high=100, shape=state.shape)
elif observation_type == 'Rgbd':
s_high = state
s_high[:, :, -1] = 100.0 # max_depth
s_high[:, :, :-1] = 255 # max_rgb
s_low = np.zeros(state.shape)
if self.use_gym_10_api:
observation_space = spaces.Box(low=s_low, high=s_high, dtype=np.float16) # for gym>=0.10
else:
observation_space = spaces.Box(low=s_low, high=s_high)
elif observation_type == 'Depth':
if self.use_gym_10_api:
observation_space = spaces.Box(low=0, high=100, shape=state.shape, dtype=np.float16) # for gym>=0.10
else:
observation_space = spaces.Box(low=0, high=100, shape=state.shape)

elif observation_type == 'Rgbd':
s_high = state
s_high[:, :, -1] = 100.0 # max_depth
s_high[:, :, :-1] = 255 # max_rgb
s_low = np.zeros(state.shape)
if self.use_gym_10_api:
observation_space = spaces.Box(low=s_low, high=s_high, dtype=np.float16) # for gym>=0.10
else:
observation_space = spaces.Box(low=s_low, high=s_high)

return observation_space

Expand All @@ -78,23 +74,23 @@ def set_texture(self, target, color=(1, 1, 1), param=(0, 0, 0), picpath=None, ti
param = param / param.max()
# color = color / color.max()
cmd = 'vbp {target} set_mat {e_num} {r} {g} {b} {meta} {spec} {rough} {tiling} {picpath}'
self.client.request(cmd.format(target=target, e_num=e_num, r=color[0], g=color[1], b=color[2],
meta=param[0], spec=param[1], rough=param[2], tiling=tiling,
picpath=picpath), -1)
res = self.client.request(cmd.format(target=target, e_num=e_num, r=color[0], g=color[1], b=color[2],
meta=param[0], spec=param[1], rough=param[2], tiling=tiling,
picpath=picpath))

def set_light(self, target, direction, intensity, color): # param num out of range
[roll, yaw, pitch] = direction
color = color / color.max()
[r, g, b] = color
cmd = f'vbp {target} set_light {roll} {yaw} {pitch} {intensity} {r} {g} {b}'
self.client.request(cmd, -1)

def set_skylight(self, obj, color, intensity ): # param num out of range
[r, g, b] = color
cmd = f'vbp {obj} set_light {r} {g} {b} {intensity} '
self.client.request(cmd, -1)

def get_pose(self, cam_id, type='hard'): # pose = [x, y, z, roll, yaw, pitch]
cmd = 'vbp {target} set_light {row} {yaw} {pitch} {intensity} {r} {g} {b}'
color = color/color.max()
res = self.client.request(cmd.format(target=target, row=direction[0], yaw=direction[1],
pitch=direction[2], intensity=intensity,
r=color[0], g=color[1], b=color[2]))

def set_skylight(self, target, color, intensity ): # param num out of range
cmd = 'vbp {target} set_light {r} {g} {b} {intensity}'
res = self.client.request(cmd.format(target=target, intensity=intensity,
r=color[0], g=color[1], b=color[2]))

def get_pose(self,cam_id, type='hard'): # pose = [x, y, z, roll, yaw, pitch]
if type == 'soft':
pose = self.cam[cam_id]['location']
pose.extend(self.cam[cam_id]['rotation'])
Expand All @@ -105,3 +101,7 @@ def get_pose(self, cam_id, type='hard'): # pose = [x, y, z, roll, yaw, pitch]
self.cam[cam_id]['rotation'] = self.get_rotation(cam_id)
pose = self.cam[cam_id]['location'] + self.cam[cam_id]['rotation']
return pose

def set_simple_texture(self, target,picpath=None):
cmd = 'vbp {target} set_mat {picpath}'
res = self.client.request(cmd.format(target=target, picpath=picpath))
69 changes: 69 additions & 0 deletions gym_unrealcv/envs/setting/tracking_multicam/MTMC.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"env_name": "MTMC16",
"env_bin": "MTMC16/LinuxNoEditor/MTMC16/Binaries/Linux/MTMC16",
"imgs_dir": "textures",
"cam_id": [0, 1, 2, 3, 4, 5],
"height": 350,
"pitch": 0,
"max_steps": 100,
"max_obstacles": 6,
"zoom": 0,
"field_size": [2696, 4174],
"target_default_z": 90,
"targets": [
"Ch01_BP_51", "Ch02_BP_124", "Ch07_BP_62", "Ch08_BP_70", "Ch21_BP_26",
"Ch22_BP_98", "Ch26_BP_115", "Ch31_BP_75", "Ch38_BP_89", "Ch42_BP_19"
],
"discrete_actions": [
[-100, -5],
[-100, 0],
[-100, 5],
[0, -5],
[0, 0],
[0, 5],
[100, -5],
[100, 0],
[100, 5]
],
"discrete_actions_player": [
[ 100, 0],
[-100, 0],
[ 50, 15],
[ 50,-15],
[ 0, 30],
[ 0, -30],
[ 0, 0]
],
"safe_start" :[
[ -1000, 1000, -1000, 1000]
],
"cam_safe_start": [
[0],
[2696],
[4783],
[6870],
[9566],
[11653]
],
"reset_area": [-1348, 1348, -2087, 2087, 50, 100],
"continous_actions": {
"high": [ 40, 15, 15],
"low": [-40, -15, -15]
},
"continous_actions_player": {
"high": [100, 30, 0],
"low": [50, -30, 0]
},
"objects_list": [
"Cone_14", "Cone2_17", "Cube_2", "Cube2_5", "Cylinder_8", "Cylinder2_11"
],
"backgrounds": [
"Floor_1","Wall7_4","Wall9","Wall10","Wall11"
],
"lights": [
],
"target_move": [100, 300],
"camera_move": [200, 200],
"scale_rate": 0 ,
"pose_rate": 0.5
}
70 changes: 70 additions & 0 deletions gym_unrealcv/envs/setting/tracking_multicam/MTMC18.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{
"env_name": "MTMC18",
"env_bin": "MTMC18/LinuxNoEditor/MTMC16/Binaries/Linux/MTMC16",
"imgs_dir": "textures",
"cam_id": [1, 2, 3, 4, 5, 6],
"height": 350,
"pitch": 0,
"max_steps": 100,
"max_obstacles": 6,
"zoom": 0,
"field_size": [2696, 4174],
"resolution": [640, 480],
"target_default_z": 90,
"targets": [
"Ch01_BP_67", "Ch02_BP_54", "Ch07_BP_70", "Ch13_BP_2", "Ch21_BP_33",
"Ch22_BP_80", "Ch26_BP_65", "Ch31_BP_69", "Ch38_BP_35", "Ch42_BP_82"
],
"discrete_actions": [
[-100, -5],
[-100, 0],
[-100, 5],
[0, -5],
[0, 0],
[0, 5],
[100, -5],
[100, 0],
[100, 5]
],
"discrete_actions_player": [
[ 100, 0],
[-100, 0],
[ 50, 15],
[ 50,-15],
[ 0, 30],
[ 0, -30],
[ 0, 0]
],
"safe_start" :[
[ -1000, 1000, -1000, 1000]
],
"cam_safe_start": [
[0],
[2696],
[4783],
[6870],
[9566],
[11653]
],
"reset_area": [-1348, 1348, -2087, 2087, 50, 100],
"continous_actions": {
"high": [ 40, 15, 15],
"low": [-40, -15, -15]
},
"continous_actions_player": {
"high": [100, 30, 0],
"low": [50, -30, 0]
},
"objects_list": [
"Cone_14", "Cone2_17", "Cube_2", "Cube2_5", "Cylinder_8", "Cylinder2_11"
],
"backgrounds": [
"Floor_1","Wall7_4","Wall9","Wall10","Wall11"
],
"lights": [
],
"target_move": [100, 300],
"camera_move": [200, 200],
"scale_rate": 0 ,
"pose_rate": 0.5
}
Loading