gameboy_worlds.emulation.deja_vu.base_metrics
1from typing import Optional 2from abc import ABC 3from gameboy_worlds.emulation.deja_vu.parsers import ( 4 AgentState, 5 # MemoryBasedDejaVuStateParser, 6 DejaVuStateParser, 7 DejaVu1StateParser, 8) 9from gameboy_worlds.emulation.tracker import ( 10 MetricGroup, 11 OCRegionMetric, 12 # TerminationTruncationMetric, 13) 14 15 16import numpy as np 17 18 19class CoreDejaVuMetrics(MetricGroup): 20 """ 21 Deja Vu-specific core metrics. 22 23 Reports: 24 - agent_state: The AgentState info. Is either Free Roam, In Dialogue or In Menu. 25 26 Final Reports: 27 - None 28 """ 29 30 NAME = "dejavu_core" 31 REQUIRED_PARSER = DejaVuStateParser 32 33 def start(self): 34 self.n_battles_total = [] 35 super().start() 36 37 def reset(self, first=False): 38 if not first: 39 pass 40 self.current_state: AgentState = ( 41 AgentState.IN_DIALOGUE 42 ) # Start by default in dialogue because it has the least permissable actions. 43 """ The current state of the agent in the game. """ 44 self._previous_state = self.current_state 45 46 def close(self): 47 self.reset() 48 return 49 50 def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]): 51 self._previous_state = self.current_state 52 current_state = self.state_parser.get_agent_state(current_frame) 53 self.current_state = current_state 54 55 def report(self) -> dict: 56 """ 57 Reports the current Deja Vu core metrics: 58 - Agent state 59 Returns: 60 dict: A dictionary containing the current agent state. 61 """ 62 return { 63 "agent_state": self.current_state, 64 } 65 66 def report_final(self) -> dict: 67 """ 68 Reports nothing: 69 """ 70 return {} 71 72 73class DejaVuOCRMetric(OCRegionMetric): 74 REQUIRED_PARSER = DejaVuStateParser 75 76 def reset(self, first=False): 77 super().reset(first) 78 self.prev_was_in_dialogue = False 79 80 def start(self): 81 self.kinds = { 82 "dialogue": "dialogue_box_area", 83 "menu": "menu_box_area", 84 } 85 super().start() 86 87 def can_read_kind(self, current_frame: np.ndarray, kind: str) -> bool: 88 self.state_parser: DejaVuStateParser 89 if kind == "dialogue": 90 in_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame) 91 # in_menu = self.state_parser.is_in_menu(current_screen=current_frame) 92 return in_dialogue # and not in_menu 93 if kind == "menu": 94 in_menu = self.state_parser.is_in_menu(current_screen=current_frame) 95 return in_menu 96 return False 97 98 99class DejaVuTestMetric(MetricGroup): 100 """ 101 Deja Vu-specific test metrics. 102 103 Reports: 104 - is_in_fight: Whether the agent is currently in a fight 105 - was_in_fight_last_step: Whether the agent was in a fight in the previous step 106 """ 107 108 NAME = "dejavu_test" 109 REQUIRED_PARSER = DejaVuStateParser 110 111 def start(self): 112 super().start() 113 114 def reset(self, first=False): 115 if not first: 116 pass 117 self.is_in_dialogue = False 118 self.pre_was_dialogue = False 119 120 def close(self): 121 self.reset() 122 return 123 124 def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]): 125 self.state_parser: DejaVuStateParser 126 is_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame) 127 self.pre_was_dialogue = self.is_in_dialogue 128 self.is_in_dialogue = is_dialogue 129 130 def report(self) -> dict: 131 return { 132 "is_in_dialogue": self.is_in_dialogue, 133 "was_in_dialogue_last_step": self.pre_was_dialogue, 134 } 135 136 def report_final(self) -> dict: 137 return {}
20class CoreDejaVuMetrics(MetricGroup): 21 """ 22 Deja Vu-specific core metrics. 23 24 Reports: 25 - agent_state: The AgentState info. Is either Free Roam, In Dialogue or In Menu. 26 27 Final Reports: 28 - None 29 """ 30 31 NAME = "dejavu_core" 32 REQUIRED_PARSER = DejaVuStateParser 33 34 def start(self): 35 self.n_battles_total = [] 36 super().start() 37 38 def reset(self, first=False): 39 if not first: 40 pass 41 self.current_state: AgentState = ( 42 AgentState.IN_DIALOGUE 43 ) # Start by default in dialogue because it has the least permissable actions. 44 """ The current state of the agent in the game. """ 45 self._previous_state = self.current_state 46 47 def close(self): 48 self.reset() 49 return 50 51 def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]): 52 self._previous_state = self.current_state 53 current_state = self.state_parser.get_agent_state(current_frame) 54 self.current_state = current_state 55 56 def report(self) -> dict: 57 """ 58 Reports the current Deja Vu core metrics: 59 - Agent state 60 Returns: 61 dict: A dictionary containing the current agent state. 62 """ 63 return { 64 "agent_state": self.current_state, 65 } 66 67 def report_final(self) -> dict: 68 """ 69 Reports nothing: 70 """ 71 return {}
Deja Vu-specific core metrics.
Reports:
- agent_state: The AgentState info. Is either Free Roam, In Dialogue or In Menu.
Final Reports:
- None
The StateParser which implements the minimum required functionality for this MetricGroup to work.
Called once when environment starts. All subclasses should call super() AFTER initializing their own variables. Only variables that will persist across episodes should be initialized here.
38 def reset(self, first=False): 39 if not first: 40 pass 41 self.current_state: AgentState = ( 42 AgentState.IN_DIALOGUE 43 ) # Start by default in dialogue because it has the least permissable actions. 44 """ The current state of the agent in the game. """ 45 self._previous_state = self.current_state
Called when environment resets.
Arguments:
- first (bool): Whether this is the first reset of the environment. If True, might need to aggregate metrics into running final totals.
Called when environment closes. Good for computing summary stats.
Step will not be called after this.
51 def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]): 52 self._previous_state = self.current_state 53 current_state = self.state_parser.get_agent_state(current_frame) 54 self.current_state = current_state
Called each environment step to update metrics.
Arguments:
- current_frame (np.ndarray): The current frame rendered by the emulator.
- recent_frames (Optional[np.ndarray]): The stack of frames that were rendered during the last action. Shape is [n_frames, height, width, channels]. Can be None if rendering is disabled.
56 def report(self) -> dict: 57 """ 58 Reports the current Deja Vu core metrics: 59 - Agent state 60 Returns: 61 dict: A dictionary containing the current agent state. 62 """ 63 return { 64 "agent_state": self.current_state, 65 }
Reports the current Deja Vu core metrics:
- Agent state
Returns:
dict: A dictionary containing the current agent state.
74class DejaVuOCRMetric(OCRegionMetric): 75 REQUIRED_PARSER = DejaVuStateParser 76 77 def reset(self, first=False): 78 super().reset(first) 79 self.prev_was_in_dialogue = False 80 81 def start(self): 82 self.kinds = { 83 "dialogue": "dialogue_box_area", 84 "menu": "menu_box_area", 85 } 86 super().start() 87 88 def can_read_kind(self, current_frame: np.ndarray, kind: str) -> bool: 89 self.state_parser: DejaVuStateParser 90 if kind == "dialogue": 91 in_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame) 92 # in_menu = self.state_parser.is_in_menu(current_screen=current_frame) 93 return in_dialogue # and not in_menu 94 if kind == "menu": 95 in_menu = self.state_parser.is_in_menu(current_screen=current_frame) 96 return in_menu 97 return False
Watch particular screen regions and capture subscreens for OCR when possible. Does not actually perform OCR itself, but makes it easy to capture the relevant regions.
Children implementing this must define self.kinds in start() and then call on super().start().
Reports:
ocr_regions: A dictionary mapping kinds to captured regions that had OCR-eligible text detected in them. The keys are kinds of OCR regions, and the values are the stacks of captured screen regions as numpy arrays of shape (num_captures, height, width, channels).step: The current step number. Useful for differentiating when multiple OCR texts were found in the same episode. You can typically safely ignore this.
Final Reports:
ocr_regions: A list of tuples for all steps where OCR was detected. Is in form:List[Tuple[int, Dict[str, np.ndarray]]]where the int is the step number and the Dict maps kinds to a stack of the captured screen region.
The StateParser which implements the minimum required functionality for this MetricGroup to work.
ocr_regions will track a list of the form List[Tuple[int, Dict[str, np.ndarray]]] which is a list of (step_number, {kind: ocr_region}) dictionaries.
81 def start(self): 82 self.kinds = { 83 "dialogue": "dialogue_box_area", 84 "menu": "menu_box_area", 85 } 86 super().start()
Assumes the child has initialized a dict called self.kinds which tracks the various kinds of OCR that could be done. self.kinds should be in the form: {kind: region_name} where region_name is the name of the region to OCR for that kind. Will track ocr captured region results in form of list of dictionaries where these kinds are keys.
88 def can_read_kind(self, current_frame: np.ndarray, kind: str) -> bool: 89 self.state_parser: DejaVuStateParser 90 if kind == "dialogue": 91 in_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame) 92 # in_menu = self.state_parser.is_in_menu(current_screen=current_frame) 93 return in_dialogue # and not in_menu 94 if kind == "menu": 95 in_menu = self.state_parser.is_in_menu(current_screen=current_frame) 96 return in_menu 97 return False
Checks if the frame has text for the given kind.
Arguments:
- frame (np.ndarray): The frame to check.
- kind (str): The kind of text to check for.
100class DejaVuTestMetric(MetricGroup): 101 """ 102 Deja Vu-specific test metrics. 103 104 Reports: 105 - is_in_fight: Whether the agent is currently in a fight 106 - was_in_fight_last_step: Whether the agent was in a fight in the previous step 107 """ 108 109 NAME = "dejavu_test" 110 REQUIRED_PARSER = DejaVuStateParser 111 112 def start(self): 113 super().start() 114 115 def reset(self, first=False): 116 if not first: 117 pass 118 self.is_in_dialogue = False 119 self.pre_was_dialogue = False 120 121 def close(self): 122 self.reset() 123 return 124 125 def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]): 126 self.state_parser: DejaVuStateParser 127 is_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame) 128 self.pre_was_dialogue = self.is_in_dialogue 129 self.is_in_dialogue = is_dialogue 130 131 def report(self) -> dict: 132 return { 133 "is_in_dialogue": self.is_in_dialogue, 134 "was_in_dialogue_last_step": self.pre_was_dialogue, 135 } 136 137 def report_final(self) -> dict: 138 return {}
Deja Vu-specific test metrics.
Reports:
- is_in_fight: Whether the agent is currently in a fight
- was_in_fight_last_step: Whether the agent was in a fight in the previous step
The StateParser which implements the minimum required functionality for this MetricGroup to work.
Called once when environment starts. All subclasses should call super() AFTER initializing their own variables. Only variables that will persist across episodes should be initialized here.
115 def reset(self, first=False): 116 if not first: 117 pass 118 self.is_in_dialogue = False 119 self.pre_was_dialogue = False
Called when environment resets.
Arguments:
- first (bool): Whether this is the first reset of the environment. If True, might need to aggregate metrics into running final totals.
Called when environment closes. Good for computing summary stats.
Step will not be called after this.
125 def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]): 126 self.state_parser: DejaVuStateParser 127 is_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame) 128 self.pre_was_dialogue = self.is_in_dialogue 129 self.is_in_dialogue = is_dialogue
Called each environment step to update metrics.
Arguments:
- current_frame (np.ndarray): The current frame rendered by the emulator.
- recent_frames (Optional[np.ndarray]): The stack of frames that were rendered during the last action. Shape is [n_frames, height, width, channels]. Can be None if rendering is disabled.
131 def report(self) -> dict: 132 return { 133 "is_in_dialogue": self.is_in_dialogue, 134 "was_in_dialogue_last_step": self.pre_was_dialogue, 135 }
Return metrics as dictionary for instantaneous variable tracking.
Returns
Dictionary of metrics