gameboy_worlds.interface.pokemon.controllers 

  1from gameboy_worlds.utils import log_error, log_info
  2from gameboy_worlds.interface.pokemon.actions import (
  3    MoveStepsAction,
  4    MenuAction,
  5    InteractAction,
  6    PassDialogueAction,
  7    BattleMenuAction,
  8    PickAttackAction,
  9    MoveGridAction,
 10    OpenMenuAction,
 11)
 12from gameboy_worlds.interface.controller import Controller
 13from gameboy_worlds.interface.action import HighLevelAction
 14from gameboy_worlds.emulation.pokemon.parsers import AgentState
 15from typing import Dict, Any
 16
 17
 18class PokemonStateWiseController(Controller):
 19    """
 20    High Level Actions for the Pokemon Environment:
 21
 22    - In Free Roam:
 23        - MoveStepsAction(direction: str, steps: int): Move in a particular direction by a specified number of grid steps.
 24        - InteractAction(): Interact with cell directly in front of you. Only works if there is something to interact with.
 25        - OpenMenuAction(option: str): Open a specific player menu option.
 26
 27    - In Dialogue:
 28        - PassDialogueAction(): Advance the dialogue by one step.
 29
 30    - In Battle:
 31        - BattleMenuAction(option: str): Navigate the battle menu to select an option. Fight to choose an attack, Pokemon to switch Pokemon, Bag to use an item, Run to attempt to flee the battle, and Progress to continue dialogue or other battle events.
 32        - In Fight Options Menu:
 33            - PickAttackAction(option: int): Select an attack option in the battle fight menu.
 34
 35    - In Menu:
 36        - MenuAction(menu_action: str): Navigate the game menu.
 37    """
 38
 39    ACTIONS = [
 40        MoveStepsAction,
 41        MenuAction,
 42        InteractAction,
 43        PassDialogueAction,
 44        BattleMenuAction,
 45        PickAttackAction,
 46        MoveStepsAction,
 47        OpenMenuAction,
 48    ]
 49
 50    def string_to_high_level_action(self, input_str):
 51        input_str = input_str.lower().strip()
 52        if "(" not in input_str or ")" not in input_str:
 53            return None, None  # Invalid format
 54        action_name = input_str.split("(")[0].strip()
 55        action_args_str = input_str.split("(")[1].split(")")[0].strip()
 56        # First handle the no arg actions
 57        if action_name == "interact":
 58            return InteractAction, {}
 59        if action_name == "passdialogue":
 60            return PassDialogueAction, {}
 61        # Now handle the actions with fixed options
 62        if action_name == "battlemenu":
 63            option = action_args_str.strip()
 64            if option in ["fight", "pokemon", "bag", "run", "progress"]:
 65                return BattleMenuAction, {"option": option}
 66            else:
 67                return None, None
 68        if action_name == "pickattack":
 69            if not action_args_str.strip().isnumeric():
 70                return None, None
 71            option = int(action_args_str.strip())
 72            if option < 1 or option > 4:
 73                return None, None
 74            return PickAttackAction, {"option": option}
 75        if action_name == "menu":
 76            option = action_args_str.strip()
 77            if option in ["up", "down", "left", "right", "confirm", "back"]:
 78                return MenuAction, {"menu_action": option}
 79            else:
 80                return None, None
 81        if action_name == "move":
 82            cardinal = None
 83            if "up" in action_args_str:
 84                cardinal = "up"
 85            if "down" in action_args_str:
 86                if cardinal is not None:
 87                    return None, None
 88                cardinal = "down"
 89            if "left" in action_args_str:
 90                if cardinal is not None:
 91                    return None, None
 92                cardinal = "left"
 93            if "right" in action_args_str:
 94                if cardinal is not None:
 95                    return None, None
 96                cardinal = "right"
 97            if cardinal is None:
 98                return None, None
 99            steps_part = action_args_str.replace(cardinal, "").strip()
100            if not steps_part.isnumeric():
101                return None, None
102            steps = int(steps_part)
103            return MoveStepsAction, {"direction": cardinal, "steps": steps}
104        if action_name == "openmenu":
105            option = action_args_str.strip()
106            return OpenMenuAction, {"option": option}
107        return None, None
108
109    def get_action_strings(
110        self, return_all: bool = False
111    ) -> Dict[HighLevelAction, str]:
112        current_state = self._emulator.state_parser.get_agent_state(
113            self._emulator.get_current_frame()
114        )
115        free_roam_action_strings = {
116            MoveStepsAction: "move(<up, down, right or left> <steps: int>): Move in a particular direction by a specified number of grid steps.",
117            InteractAction: "interact(): Interact with cell directly in front of you. Only works if there is something to interact with.",
118            OpenMenuAction: "openmenu(<pokedex, pokemon, bag, trainer>): Open a specific player menu option.",
119        }
120        dialogue_action_strings = {
121            PassDialogueAction: "passdialogue(): Advance the dialogue by one step.",
122        }
123        battle_action_strings = {
124            BattleMenuAction: "battlemenu(<fight, pokemon, bag, run or progress>): Navigate the battle menu to select an option. Fight to choose an attack, Pokemon to switch Pokemon, Bag to use an item, Run to attempt to flee the battle, and Progress to continue dialogue or other battle events.",
125        }
126        pick_attack_action_strings = {
127            PickAttackAction: "pickattack(<1-4>): Select an attack option in the battle fight menu.",
128        }
129        menu_action_strings = {
130            MenuAction: "menu(<up, down, left, right, confirm or back>): Navigate the game menu.",
131        }
132        if return_all:
133            actions = {
134                **free_roam_action_strings,
135                **dialogue_action_strings,
136                **battle_action_strings,
137                **pick_attack_action_strings,
138                **menu_action_strings,
139            }
140        else:
141            if current_state == AgentState.FREE_ROAM:
142                actions = free_roam_action_strings
143            elif current_state == AgentState.IN_DIALOGUE:
144                actions = dialogue_action_strings
145            elif current_state == AgentState.IN_BATTLE:
146                if self._emulator.state_parser.is_in_fight_options_menu(
147                    self._emulator.get_current_frame()
148                ):
149                    actions = {**battle_action_strings, **pick_attack_action_strings}
150                else:
151                    actions = battle_action_strings
152            elif current_state == AgentState.IN_MENU:
153                actions = menu_action_strings
154            else:
155                log_error(
156                    f"Unknown agent state {current_state} when getting action strings."
157                )
158        return actions
class PokemonStateWiseController(gameboy_worlds.interface.controller.Controller): 
 19class PokemonStateWiseController(Controller):
 20    """
 21    High Level Actions for the Pokemon Environment:
 22
 23    - In Free Roam:
 24        - MoveStepsAction(direction: str, steps: int): Move in a particular direction by a specified number of grid steps.
 25        - InteractAction(): Interact with cell directly in front of you. Only works if there is something to interact with.
 26        - OpenMenuAction(option: str): Open a specific player menu option.
 27
 28    - In Dialogue:
 29        - PassDialogueAction(): Advance the dialogue by one step.
 30
 31    - In Battle:
 32        - BattleMenuAction(option: str): Navigate the battle menu to select an option. Fight to choose an attack, Pokemon to switch Pokemon, Bag to use an item, Run to attempt to flee the battle, and Progress to continue dialogue or other battle events.
 33        - In Fight Options Menu:
 34            - PickAttackAction(option: int): Select an attack option in the battle fight menu.
 35
 36    - In Menu:
 37        - MenuAction(menu_action: str): Navigate the game menu.
 38    """
 39
 40    ACTIONS = [
 41        MoveStepsAction,
 42        MenuAction,
 43        InteractAction,
 44        PassDialogueAction,
 45        BattleMenuAction,
 46        PickAttackAction,
 47        MoveStepsAction,
 48        OpenMenuAction,
 49    ]
 50
 51    def string_to_high_level_action(self, input_str):
 52        input_str = input_str.lower().strip()
 53        if "(" not in input_str or ")" not in input_str:
 54            return None, None  # Invalid format
 55        action_name = input_str.split("(")[0].strip()
 56        action_args_str = input_str.split("(")[1].split(")")[0].strip()
 57        # First handle the no arg actions
 58        if action_name == "interact":
 59            return InteractAction, {}
 60        if action_name == "passdialogue":
 61            return PassDialogueAction, {}
 62        # Now handle the actions with fixed options
 63        if action_name == "battlemenu":
 64            option = action_args_str.strip()
 65            if option in ["fight", "pokemon", "bag", "run", "progress"]:
 66                return BattleMenuAction, {"option": option}
 67            else:
 68                return None, None
 69        if action_name == "pickattack":
 70            if not action_args_str.strip().isnumeric():
 71                return None, None
 72            option = int(action_args_str.strip())
 73            if option < 1 or option > 4:
 74                return None, None
 75            return PickAttackAction, {"option": option}
 76        if action_name == "menu":
 77            option = action_args_str.strip()
 78            if option in ["up", "down", "left", "right", "confirm", "back"]:
 79                return MenuAction, {"menu_action": option}
 80            else:
 81                return None, None
 82        if action_name == "move":
 83            cardinal = None
 84            if "up" in action_args_str:
 85                cardinal = "up"
 86            if "down" in action_args_str:
 87                if cardinal is not None:
 88                    return None, None
 89                cardinal = "down"
 90            if "left" in action_args_str:
 91                if cardinal is not None:
 92                    return None, None
 93                cardinal = "left"
 94            if "right" in action_args_str:
 95                if cardinal is not None:
 96                    return None, None
 97                cardinal = "right"
 98            if cardinal is None:
 99                return None, None
100            steps_part = action_args_str.replace(cardinal, "").strip()
101            if not steps_part.isnumeric():
102                return None, None
103            steps = int(steps_part)
104            return MoveStepsAction, {"direction": cardinal, "steps": steps}
105        if action_name == "openmenu":
106            option = action_args_str.strip()
107            return OpenMenuAction, {"option": option}
108        return None, None
109
110    def get_action_strings(
111        self, return_all: bool = False
112    ) -> Dict[HighLevelAction, str]:
113        current_state = self._emulator.state_parser.get_agent_state(
114            self._emulator.get_current_frame()
115        )
116        free_roam_action_strings = {
117            MoveStepsAction: "move(<up, down, right or left> <steps: int>): Move in a particular direction by a specified number of grid steps.",
118            InteractAction: "interact(): Interact with cell directly in front of you. Only works if there is something to interact with.",
119            OpenMenuAction: "openmenu(<pokedex, pokemon, bag, trainer>): Open a specific player menu option.",
120        }
121        dialogue_action_strings = {
122            PassDialogueAction: "passdialogue(): Advance the dialogue by one step.",
123        }
124        battle_action_strings = {
125            BattleMenuAction: "battlemenu(<fight, pokemon, bag, run or progress>): Navigate the battle menu to select an option. Fight to choose an attack, Pokemon to switch Pokemon, Bag to use an item, Run to attempt to flee the battle, and Progress to continue dialogue or other battle events.",
126        }
127        pick_attack_action_strings = {
128            PickAttackAction: "pickattack(<1-4>): Select an attack option in the battle fight menu.",
129        }
130        menu_action_strings = {
131            MenuAction: "menu(<up, down, left, right, confirm or back>): Navigate the game menu.",
132        }
133        if return_all:
134            actions = {
135                **free_roam_action_strings,
136                **dialogue_action_strings,
137                **battle_action_strings,
138                **pick_attack_action_strings,
139                **menu_action_strings,
140            }
141        else:
142            if current_state == AgentState.FREE_ROAM:
143                actions = free_roam_action_strings
144            elif current_state == AgentState.IN_DIALOGUE:
145                actions = dialogue_action_strings
146            elif current_state == AgentState.IN_BATTLE:
147                if self._emulator.state_parser.is_in_fight_options_menu(
148                    self._emulator.get_current_frame()
149                ):
150                    actions = {**battle_action_strings, **pick_attack_action_strings}
151                else:
152                    actions = battle_action_strings
153            elif current_state == AgentState.IN_MENU:
154                actions = menu_action_strings
155            else:
156                log_error(
157                    f"Unknown agent state {current_state} when getting action strings."
158                )
159        return actions

High Level Actions for the Pokemon Environment:

  • In Free Roam:

    • MoveStepsAction(direction: str, steps: int): Move in a particular direction by a specified number of grid steps.
    • InteractAction(): Interact with cell directly in front of you. Only works if there is something to interact with.
    • OpenMenuAction(option: str): Open a specific player menu option.

  • In Dialogue:

    • PassDialogueAction(): Advance the dialogue by one step.

  • In Battle:

    • BattleMenuAction(option: str): Navigate the battle menu to select an option. Fight to choose an attack, Pokemon to switch Pokemon, Bag to use an item, Run to attempt to flee the battle, and Progress to continue dialogue or other battle events.
    • In Fight Options Menu:
      • PickAttackAction(option: int): Select an attack option in the battle fight menu.

  • In Menu:

    • MenuAction(menu_action: str): Navigate the game menu.
def string_to_high_level_action(self, input_str): 
 51    def string_to_high_level_action(self, input_str):
 52        input_str = input_str.lower().strip()
 53        if "(" not in input_str or ")" not in input_str:
 54            return None, None  # Invalid format
 55        action_name = input_str.split("(")[0].strip()
 56        action_args_str = input_str.split("(")[1].split(")")[0].strip()
 57        # First handle the no arg actions
 58        if action_name == "interact":
 59            return InteractAction, {}
 60        if action_name == "passdialogue":
 61            return PassDialogueAction, {}
 62        # Now handle the actions with fixed options
 63        if action_name == "battlemenu":
 64            option = action_args_str.strip()
 65            if option in ["fight", "pokemon", "bag", "run", "progress"]:
 66                return BattleMenuAction, {"option": option}
 67            else:
 68                return None, None
 69        if action_name == "pickattack":
 70            if not action_args_str.strip().isnumeric():
 71                return None, None
 72            option = int(action_args_str.strip())
 73            if option < 1 or option > 4:
 74                return None, None
 75            return PickAttackAction, {"option": option}
 76        if action_name == "menu":
 77            option = action_args_str.strip()
 78            if option in ["up", "down", "left", "right", "confirm", "back"]:
 79                return MenuAction, {"menu_action": option}
 80            else:
 81                return None, None
 82        if action_name == "move":
 83            cardinal = None
 84            if "up" in action_args_str:
 85                cardinal = "up"
 86            if "down" in action_args_str:
 87                if cardinal is not None:
 88                    return None, None
 89                cardinal = "down"
 90            if "left" in action_args_str:
 91                if cardinal is not None:
 92                    return None, None
 93                cardinal = "left"
 94            if "right" in action_args_str:
 95                if cardinal is not None:
 96                    return None, None
 97                cardinal = "right"
 98            if cardinal is None:
 99                return None, None
100            steps_part = action_args_str.replace(cardinal, "").strip()
101            if not steps_part.isnumeric():
102                return None, None
103            steps = int(steps_part)
104            return MoveStepsAction, {"direction": cardinal, "steps": steps}
105        if action_name == "openmenu":
106            option = action_args_str.strip()
107            return OpenMenuAction, {"option": option}
108        return None, None

Provide a way to map a string input to a HighLevelAction and parameters.

Implement if you want to use the human_step_play method, or if you want to allow a LM based agent to give its actions in text. Must return None, None if the input_str does not map to an action.

def get_action_strings( self, return_all: bool = False) -> Dict[gameboy_worlds.interface.action.HighLevelAction, str]: 
110    def get_action_strings(
111        self, return_all: bool = False
112    ) -> Dict[HighLevelAction, str]:
113        current_state = self._emulator.state_parser.get_agent_state(
114            self._emulator.get_current_frame()
115        )
116        free_roam_action_strings = {
117            MoveStepsAction: "move(<up, down, right or left> <steps: int>): Move in a particular direction by a specified number of grid steps.",
118            InteractAction: "interact(): Interact with cell directly in front of you. Only works if there is something to interact with.",
119            OpenMenuAction: "openmenu(<pokedex, pokemon, bag, trainer>): Open a specific player menu option.",
120        }
121        dialogue_action_strings = {
122            PassDialogueAction: "passdialogue(): Advance the dialogue by one step.",
123        }
124        battle_action_strings = {
125            BattleMenuAction: "battlemenu(<fight, pokemon, bag, run or progress>): Navigate the battle menu to select an option. Fight to choose an attack, Pokemon to switch Pokemon, Bag to use an item, Run to attempt to flee the battle, and Progress to continue dialogue or other battle events.",
126        }
127        pick_attack_action_strings = {
128            PickAttackAction: "pickattack(<1-4>): Select an attack option in the battle fight menu.",
129        }
130        menu_action_strings = {
131            MenuAction: "menu(<up, down, left, right, confirm or back>): Navigate the game menu.",
132        }
133        if return_all:
134            actions = {
135                **free_roam_action_strings,
136                **dialogue_action_strings,
137                **battle_action_strings,
138                **pick_attack_action_strings,
139                **menu_action_strings,
140            }
141        else:
142            if current_state == AgentState.FREE_ROAM:
143                actions = free_roam_action_strings
144            elif current_state == AgentState.IN_DIALOGUE:
145                actions = dialogue_action_strings
146            elif current_state == AgentState.IN_BATTLE:
147                if self._emulator.state_parser.is_in_fight_options_menu(
148                    self._emulator.get_current_frame()
149                ):
150                    actions = {**battle_action_strings, **pick_attack_action_strings}
151                else:
152                    actions = battle_action_strings
153            elif current_state == AgentState.IN_MENU:
154                actions = menu_action_strings
155            else:
156                log_error(
157                    f"Unknown agent state {current_state} when getting action strings."
158                )
159        return actions

Provide a way to verbalize the allowed high level actions, along with the format of the input parameters. Useful for prompting a VLM to choose an action.

This should match the mapping in string_to_high_level_action

Parameters
  • return_all: If True, returns all possible actions and parameter formats. If False, returns only the actions that are valid in the current state.
Returns

A dictionary mapping high level actions to their verbalizations and input formats.

Inherited Members
gameboy_worlds.interface.controller.Controller
Controller
actions
REQUIRED_STATE_TRACKER
action_space
seed
unassign_emulator
assign_emulator
get_action_space
sample
is_valid
get_valid_high_level_actions
get_valid_space_actions
get_possibly_valid_high_level_actions
execute_space_action
execute
string_to_space_action
execute_string