gameboy_worlds.emulation.deja_vu.base_metrics

  1from typing import Optional
  2from abc import ABC
  3from gameboy_worlds.emulation.deja_vu.parsers import (
  4    AgentState,
  5    # MemoryBasedDejaVuStateParser,
  6    DejaVuStateParser,
  7    DejaVu1StateParser,
  8)
  9from gameboy_worlds.emulation.tracker import (
 10    MetricGroup,
 11    OCRegionMetric,
 12    # TerminationTruncationMetric,
 13)
 14
 15
 16import numpy as np
 17
 18
 19class CoreDejaVuMetrics(MetricGroup):
 20    """
 21    Deja Vu-specific core metrics.
 22
 23    Reports:
 24    - agent_state: The AgentState info. Is either Free Roam, In Dialogue or In Menu.
 25
 26    Final Reports:
 27    - None
 28    """
 29
 30    NAME = "dejavu_core"
 31    REQUIRED_PARSER = DejaVuStateParser
 32
 33    def start(self):
 34        self.n_battles_total = []
 35        super().start()
 36
 37    def reset(self, first=False):
 38        if not first:
 39            pass
 40        self.current_state: AgentState = (
 41            AgentState.IN_DIALOGUE
 42        )  # Start by default in dialogue because it has the least permissable actions.
 43        """ The current state of the agent in the game. """
 44        self._previous_state = self.current_state
 45
 46    def close(self):
 47        self.reset()
 48        return
 49
 50    def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]):
 51        self._previous_state = self.current_state
 52        current_state = self.state_parser.get_agent_state(current_frame)
 53        self.current_state = current_state
 54
 55    def report(self) -> dict:
 56        """
 57        Reports the current Deja Vu core metrics:
 58        - Agent state
 59        Returns:
 60            dict: A dictionary containing the current agent state.
 61        """
 62        return {
 63            "agent_state": self.current_state,
 64        }
 65
 66    def report_final(self) -> dict:
 67        """
 68        Reports nothing:
 69        """
 70        return {}
 71
 72
 73class DejaVuOCRMetric(OCRegionMetric):
 74    REQUIRED_PARSER = DejaVuStateParser
 75
 76    def reset(self, first=False):
 77        super().reset(first)
 78        self.prev_was_in_dialogue = False
 79
 80    def start(self):
 81        self.kinds = {
 82            "dialogue": "dialogue_box_area",
 83            "menu": "menu_box_area",
 84        }
 85        super().start()
 86
 87    def can_read_kind(self, current_frame: np.ndarray, kind: str) -> bool:
 88        self.state_parser: DejaVuStateParser
 89        if kind == "dialogue":
 90            in_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame)
 91            # in_menu = self.state_parser.is_in_menu(current_screen=current_frame)
 92            return in_dialogue  # and not in_menu
 93        if kind == "menu":
 94            in_menu = self.state_parser.is_in_menu(current_screen=current_frame)
 95            return in_menu
 96        return False
 97
 98
 99class DejaVuTestMetric(MetricGroup):
100    """
101    Deja Vu-specific test metrics.
102
103    Reports:
104    - is_in_fight: Whether the agent is currently in a fight
105    - was_in_fight_last_step: Whether the agent was in a fight in the previous step
106    """
107
108    NAME = "dejavu_test"
109    REQUIRED_PARSER = DejaVuStateParser
110
111    def start(self):
112        super().start()
113
114    def reset(self, first=False):
115        if not first:
116            pass
117        self.is_in_dialogue = False
118        self.pre_was_dialogue = False
119
120    def close(self):
121        self.reset()
122        return
123
124    def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]):
125        self.state_parser: DejaVuStateParser
126        is_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame)
127        self.pre_was_dialogue = self.is_in_dialogue
128        self.is_in_dialogue = is_dialogue
129
130    def report(self) -> dict:
131        return {
132            "is_in_dialogue": self.is_in_dialogue,
133            "was_in_dialogue_last_step": self.pre_was_dialogue,
134        }
135
136    def report_final(self) -> dict:
137        return {}
class CoreDejaVuMetrics(gameboy_worlds.emulation.tracker.MetricGroup):
20class CoreDejaVuMetrics(MetricGroup):
21    """
22    Deja Vu-specific core metrics.
23
24    Reports:
25    - agent_state: The AgentState info. Is either Free Roam, In Dialogue or In Menu.
26
27    Final Reports:
28    - None
29    """
30
31    NAME = "dejavu_core"
32    REQUIRED_PARSER = DejaVuStateParser
33
34    def start(self):
35        self.n_battles_total = []
36        super().start()
37
38    def reset(self, first=False):
39        if not first:
40            pass
41        self.current_state: AgentState = (
42            AgentState.IN_DIALOGUE
43        )  # Start by default in dialogue because it has the least permissable actions.
44        """ The current state of the agent in the game. """
45        self._previous_state = self.current_state
46
47    def close(self):
48        self.reset()
49        return
50
51    def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]):
52        self._previous_state = self.current_state
53        current_state = self.state_parser.get_agent_state(current_frame)
54        self.current_state = current_state
55
56    def report(self) -> dict:
57        """
58        Reports the current Deja Vu core metrics:
59        - Agent state
60        Returns:
61            dict: A dictionary containing the current agent state.
62        """
63        return {
64            "agent_state": self.current_state,
65        }
66
67    def report_final(self) -> dict:
68        """
69        Reports nothing:
70        """
71        return {}

Deja Vu-specific core metrics.

Reports:

  • agent_state: The AgentState info. Is either Free Roam, In Dialogue or In Menu.

Final Reports:

  • None
NAME = 'dejavu_core'

Name of the MetricGroup.

The StateParser which implements the minimum required functionality for this MetricGroup to work.

def start(self):
34    def start(self):
35        self.n_battles_total = []
36        super().start()

Called once when environment starts. All subclasses should call super() AFTER initializing their own variables. Only variables that will persist across episodes should be initialized here.

def reset(self, first=False):
38    def reset(self, first=False):
39        if not first:
40            pass
41        self.current_state: AgentState = (
42            AgentState.IN_DIALOGUE
43        )  # Start by default in dialogue because it has the least permissable actions.
44        """ The current state of the agent in the game. """
45        self._previous_state = self.current_state

Called when environment resets.

Arguments:
  • first (bool): Whether this is the first reset of the environment. If True, might need to aggregate metrics into running final totals.
def close(self):
47    def close(self):
48        self.reset()
49        return

Called when environment closes. Good for computing summary stats.

Step will not be called after this.

def step( self, current_frame: numpy.ndarray, recent_frames: Optional[numpy.ndarray]):
51    def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]):
52        self._previous_state = self.current_state
53        current_state = self.state_parser.get_agent_state(current_frame)
54        self.current_state = current_state

Called each environment step to update metrics.

Arguments:
  • current_frame (np.ndarray): The current frame rendered by the emulator.
  • recent_frames (Optional[np.ndarray]): The stack of frames that were rendered during the last action. Shape is [n_frames, height, width, channels]. Can be None if rendering is disabled.
def report(self) -> dict:
56    def report(self) -> dict:
57        """
58        Reports the current Deja Vu core metrics:
59        - Agent state
60        Returns:
61            dict: A dictionary containing the current agent state.
62        """
63        return {
64            "agent_state": self.current_state,
65        }

Reports the current Deja Vu core metrics:

  • Agent state
Returns:

dict: A dictionary containing the current agent state.

def report_final(self) -> dict:
67    def report_final(self) -> dict:
68        """
69        Reports nothing:
70        """
71        return {}

Reports nothing:

class DejaVuOCRMetric(gameboy_worlds.emulation.tracker.OCRegionMetric):
74class DejaVuOCRMetric(OCRegionMetric):
75    REQUIRED_PARSER = DejaVuStateParser
76
77    def reset(self, first=False):
78        super().reset(first)
79        self.prev_was_in_dialogue = False
80
81    def start(self):
82        self.kinds = {
83            "dialogue": "dialogue_box_area",
84            "menu": "menu_box_area",
85        }
86        super().start()
87
88    def can_read_kind(self, current_frame: np.ndarray, kind: str) -> bool:
89        self.state_parser: DejaVuStateParser
90        if kind == "dialogue":
91            in_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame)
92            # in_menu = self.state_parser.is_in_menu(current_screen=current_frame)
93            return in_dialogue  # and not in_menu
94        if kind == "menu":
95            in_menu = self.state_parser.is_in_menu(current_screen=current_frame)
96            return in_menu
97        return False

Watch particular screen regions and capture subscreens for OCR when possible. Does not actually perform OCR itself, but makes it easy to capture the relevant regions. Children implementing this must define self.kinds in start() and then call on super().start().

Reports:

  • ocr_regions: A dictionary mapping kinds to captured regions that had OCR-eligible text detected in them. The keys are kinds of OCR regions, and the values are the stacks of captured screen regions as numpy arrays of shape (num_captures, height, width, channels).
  • step: The current step number. Useful for differentiating when multiple OCR texts were found in the same episode. You can typically safely ignore this.

Final Reports:

  • ocr_regions: A list of tuples for all steps where OCR was detected. Is in form: List[Tuple[int, Dict[str, np.ndarray]]] where the int is the step number and the Dict maps kinds to a stack of the captured screen region.

The StateParser which implements the minimum required functionality for this MetricGroup to work.

def reset(self, first=False):
77    def reset(self, first=False):
78        super().reset(first)
79        self.prev_was_in_dialogue = False

ocr_regions will track a list of the form List[Tuple[int, Dict[str, np.ndarray]]] which is a list of (step_number, {kind: ocr_region}) dictionaries.

def start(self):
81    def start(self):
82        self.kinds = {
83            "dialogue": "dialogue_box_area",
84            "menu": "menu_box_area",
85        }
86        super().start()

Assumes the child has initialized a dict called self.kinds which tracks the various kinds of OCR that could be done. self.kinds should be in the form: {kind: region_name} where region_name is the name of the region to OCR for that kind. Will track ocr captured region results in form of list of dictionaries where these kinds are keys.

def can_read_kind(self, current_frame: numpy.ndarray, kind: str) -> bool:
88    def can_read_kind(self, current_frame: np.ndarray, kind: str) -> bool:
89        self.state_parser: DejaVuStateParser
90        if kind == "dialogue":
91            in_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame)
92            # in_menu = self.state_parser.is_in_menu(current_screen=current_frame)
93            return in_dialogue  # and not in_menu
94        if kind == "menu":
95            in_menu = self.state_parser.is_in_menu(current_screen=current_frame)
96            return in_menu
97        return False

Checks if the frame has text for the given kind.

Arguments:
  • frame (np.ndarray): The frame to check.
  • kind (str): The kind of text to check for.
class DejaVuTestMetric(gameboy_worlds.emulation.tracker.MetricGroup):
100class DejaVuTestMetric(MetricGroup):
101    """
102    Deja Vu-specific test metrics.
103
104    Reports:
105    - is_in_fight: Whether the agent is currently in a fight
106    - was_in_fight_last_step: Whether the agent was in a fight in the previous step
107    """
108
109    NAME = "dejavu_test"
110    REQUIRED_PARSER = DejaVuStateParser
111
112    def start(self):
113        super().start()
114
115    def reset(self, first=False):
116        if not first:
117            pass
118        self.is_in_dialogue = False
119        self.pre_was_dialogue = False
120
121    def close(self):
122        self.reset()
123        return
124
125    def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]):
126        self.state_parser: DejaVuStateParser
127        is_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame)
128        self.pre_was_dialogue = self.is_in_dialogue
129        self.is_in_dialogue = is_dialogue
130
131    def report(self) -> dict:
132        return {
133            "is_in_dialogue": self.is_in_dialogue,
134            "was_in_dialogue_last_step": self.pre_was_dialogue,
135        }
136
137    def report_final(self) -> dict:
138        return {}

Deja Vu-specific test metrics.

Reports:

  • is_in_fight: Whether the agent is currently in a fight
  • was_in_fight_last_step: Whether the agent was in a fight in the previous step
NAME = 'dejavu_test'

Name of the MetricGroup.

The StateParser which implements the minimum required functionality for this MetricGroup to work.

def start(self):
112    def start(self):
113        super().start()

Called once when environment starts. All subclasses should call super() AFTER initializing their own variables. Only variables that will persist across episodes should be initialized here.

def reset(self, first=False):
115    def reset(self, first=False):
116        if not first:
117            pass
118        self.is_in_dialogue = False
119        self.pre_was_dialogue = False

Called when environment resets.

Arguments:
  • first (bool): Whether this is the first reset of the environment. If True, might need to aggregate metrics into running final totals.
def close(self):
121    def close(self):
122        self.reset()
123        return

Called when environment closes. Good for computing summary stats.

Step will not be called after this.

def step( self, current_frame: numpy.ndarray, recent_frames: Optional[numpy.ndarray]):
125    def step(self, current_frame: np.ndarray, recent_frames: Optional[np.ndarray]):
126        self.state_parser: DejaVuStateParser
127        is_dialogue = self.state_parser.is_in_dialogue(current_screen=current_frame)
128        self.pre_was_dialogue = self.is_in_dialogue
129        self.is_in_dialogue = is_dialogue

Called each environment step to update metrics.

Arguments:
  • current_frame (np.ndarray): The current frame rendered by the emulator.
  • recent_frames (Optional[np.ndarray]): The stack of frames that were rendered during the last action. Shape is [n_frames, height, width, channels]. Can be None if rendering is disabled.
def report(self) -> dict:
131    def report(self) -> dict:
132        return {
133            "is_in_dialogue": self.is_in_dialogue,
134            "was_in_dialogue_last_step": self.pre_was_dialogue,
135        }

Return metrics as dictionary for instantaneous variable tracking.

Returns

Dictionary of metrics

def report_final(self) -> dict:
137    def report_final(self) -> dict:
138        return {}

Return metrics as dictionary for logging. Called at end of environment (before close). Will never be called before self.close.

Returns

Dictionary of metrics