diff options
33 files changed, 11971 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cb33e9d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Never commit configuration files +.config + +__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a45d3e5 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +Software used/included: + +* kconfiglib: https://github.com/ulfalizer/Kconfiglib ISC License +* esptool.py: https://github.com/espressif/esptool GPLv2 +* micropython: http://micropython.org/ MIT License diff --git a/anzeige0/Kconfig b/anzeige0/Kconfig new file mode 100644 index 0000000..c588b68 --- /dev/null +++ b/anzeige0/Kconfig @@ -0,0 +1,4 @@ +mainmenu "configuration" + +source ../scripts/Kconfig-wifi + diff --git a/anzeige0/boot.py b/anzeige0/boot.py new file mode 100644 index 0000000..0d30893 --- /dev/null +++ b/anzeige0/boot.py @@ -0,0 +1,2 @@ +config = dict(map(lambda x: (x[0], int(x[1]) if x[1].isdigit() else x[1][1:-1] if x[1].startswith('"') and x[1].endswith('"') else x[1]), map(lambda x: x.strip().split("=", 1), filter(lambda x: '=' in x, open(".config").readlines())))) + diff --git a/anzeige0/lcd_api.py b/anzeige0/lcd_api.py new file mode 100644 index 0000000..38b60da --- /dev/null +++ b/anzeige0/lcd_api.py @@ -0,0 +1,195 @@ +"""Provides an API for talking to HD44780 compatible character LCDs.""" + +import time + +class LcdApi: + """Implements the API for talking with HD44780 compatible character LCDs. + This class only knows what commands to send to the LCD, and not how to get + them to the LCD. + + It is expected that a derived class will implement the hal_xxx functions. + """ + + # The following constant names were lifted from the avrlib lcd.h + # header file, however, I changed the definitions from bit numbers + # to bit masks. + # + # HD44780 LCD controller command set + + LCD_CLR = 0x01 # DB0: clear display + LCD_HOME = 0x02 # DB1: return to home position + + LCD_ENTRY_MODE = 0x04 # DB2: set entry mode + LCD_ENTRY_INC = 0x02 # --DB1: increment + LCD_ENTRY_SHIFT = 0x01 # --DB0: shift + + LCD_ON_CTRL = 0x08 # DB3: turn lcd/cursor on + LCD_ON_DISPLAY = 0x04 # --DB2: turn display on + LCD_ON_CURSOR = 0x02 # --DB1: turn cursor on + LCD_ON_BLINK = 0x01 # --DB0: blinking cursor + + LCD_MOVE = 0x10 # DB4: move cursor/display + LCD_MOVE_DISP = 0x08 # --DB3: move display (0-> move cursor) + LCD_MOVE_RIGHT = 0x04 # --DB2: move right (0-> left) + + LCD_FUNCTION = 0x20 # DB5: function set + LCD_FUNCTION_8BIT = 0x10 # --DB4: set 8BIT mode (0->4BIT mode) + LCD_FUNCTION_2LINES = 0x08 # --DB3: two lines (0->one line) + LCD_FUNCTION_10DOTS = 0x04 # --DB2: 5x10 font (0->5x7 font) + LCD_FUNCTION_RESET = 0x30 # See "Initializing by Instruction" section + + LCD_CGRAM = 0x40 # DB6: set CG RAM address + LCD_DDRAM = 0x80 # DB7: set DD RAM address + + LCD_RS_CMD = 0 + LCD_RS_DATA = 1 + + LCD_RW_WRITE = 0 + LCD_RW_READ = 1 + + def __init__(self, num_lines, num_columns): + self.num_lines = num_lines + if self.num_lines > 4: + self.num_lines = 4 + self.num_columns = num_columns + if self.num_columns > 40: + self.num_columns = 40 + self.cursor_x = 0 + self.cursor_y = 0 + self.backlight = True + self.display_off() + self.backlight_on() + self.clear() + self.hal_write_command(self.LCD_ENTRY_MODE | self.LCD_ENTRY_INC) + self.hide_cursor() + self.display_on() + + def clear(self): + """Clears the LCD display and moves the cursor to the top left + corner. + """ + self.hal_write_command(self.LCD_CLR) + self.hal_write_command(self.LCD_HOME) + self.cursor_x = 0 + self.cursor_y = 0 + + def show_cursor(self): + """Causes the cursor to be made visible.""" + self.hal_write_command(self.LCD_ON_CTRL | self.LCD_ON_DISPLAY | + self.LCD_ON_CURSOR) + + def hide_cursor(self): + """Causes the cursor to be hidden.""" + self.hal_write_command(self.LCD_ON_CTRL | self.LCD_ON_DISPLAY) + + def blink_cursor_on(self): + """Turns on the cursor, and makes it blink.""" + self.hal_write_command(self.LCD_ON_CTRL | self.LCD_ON_DISPLAY | + self.LCD_ON_CURSOR | self.LCD_ON_BLINK) + + def blink_cursor_off(self): + """Turns on the cursor, and makes it no blink (i.e. be solid).""" + self.hal_write_command(self.LCD_ON_CTRL | self.LCD_ON_DISPLAY | + self.LCD_ON_CURSOR) + + def display_on(self): + """Turns on (i.e. unblanks) the LCD.""" + self.hal_write_command(self.LCD_ON_CTRL | self.LCD_ON_DISPLAY) + + def display_off(self): + """Turns off (i.e. blanks) the LCD.""" + self.hal_write_command(self.LCD_ON_CTRL) + + def backlight_on(self): + """Turns the backlight on. + + This isn't really an LCD command, but some modules have backlight + controls, so this allows the hal to pass through the command. + """ + self.backlight = True + self.hal_backlight_on() + + def backlight_off(self): + """Turns the backlight off. + + This isn't really an LCD command, but some modules have backlight + controls, so this allows the hal to pass through the command. + """ + self.backlight = False + self.hal_backlight_off() + + def move_to(self, cursor_x, cursor_y): + """Moves the cursor position to the indicated position. The cursor + position is zero based (i.e. cursor_x == 0 indicates first column). + """ + self.cursor_x = cursor_x + self.cursor_y = cursor_y + addr = cursor_x & 0x3f + if cursor_y & 1: + addr += 0x40 # Lines 1 & 3 add 0x40 + if cursor_y & 2: + addr += 0x14 # Lines 2 & 3 add 0x14 + self.hal_write_command(self.LCD_DDRAM | addr) + + def putchar(self, char): + """Writes the indicated character to the LCD at the current cursor + position, and advances the cursor by one position. + """ + if char != '\n': + self.hal_write_data(ord(char)) + self.cursor_x += 1 + if self.cursor_x >= self.num_columns or char == '\n': + self.cursor_x = 0 + self.cursor_y += 1 + if self.cursor_y >= self.num_lines: + self.cursor_y = 0 + self.move_to(self.cursor_x, self.cursor_y) + + def putstr(self, string): + """Write the indicated string to the LCD at the current cursor + position and advances the cursor position appropriately. + """ + for char in string: + self.putchar(char) + + def custom_char(self, location, charmap): + """Write a character to one of the 8 CGRAM locations, available + as chr(0) through chr(7). + """ + location &= 0x7 + self.hal_write_command(self.LCD_CGRAM | (location << 3)) + time.sleep_us(40) + for i in range(8): + self.hal_write_data(charmap[i]) + time.sleep_us(40) + self.move_to(self.cursor_x, self.cursor_y) + + def hal_backlight_on(self): + """Allows the hal layer to turn the backlight on. + + If desired, a derived HAL class will implement this function. + """ + pass + + def hal_backlight_off(self): + """Allows the hal layer to turn the backlight off. + + If desired, a derived HAL class will implement this function. + """ + pass + + def hal_write_command(self, cmd): + """Write a command to the LCD. + + It is expected that a derived HAL class will implement this + function. + """ + raise NotImplementedError + + def hal_write_data(self, data): + """Write data to the LCD. + + It is expected that a derived HAL class will implement this + function. + """ + raise NotImplementedError diff --git a/anzeige0/main.py b/anzeige0/main.py new file mode 100644 index 0000000..c73e847 --- /dev/null +++ b/anzeige0/main.py @@ -0,0 +1,174 @@ +#!micropython +print("Start main.py") +import json +import time +import machine +import socket +import micropython as mp + +from nodemcu_gpio_lcd import GpioLcd +from machine import Pin, PWM +import network + +backlight = PWM(Pin(12), freq=500, duty=200) + +wlan = network.WLAN() + +lcd = GpioLcd(rs_pin=Pin(16), enable_pin=Pin(5), + d4_pin=Pin(4), d5_pin=Pin(0), d6_pin=Pin(2), d7_pin=Pin(14), + num_lines=2, num_columns=16) + +lcd.clear() +lcd.putstr("Verbinde mit:\n{}".format(wlan.config('essid'))) +while not wlan.isconnected(): + continue +lcd.clear() +lcd.putstr("Verbunden") + +HOST = b'www.localnet.cc' +HEADERS = b'Host: www.localnet.cc\r\nConnection: close\r\n\r\n' + +displays = [ + { # ################ ################ <- 16 chars + "name": "Zusammenfassung " + " ", + "text": "I: ____ E: ____ " + "W: ____ßS: ____ß", + "values" : [ + """SELECT sum("d4_count")*30 FROM "stromzaehler3" WHERE time > now() - 2m""", + """SELECT sum("d1_count")*30 FROM "stromzaehler3" WHERE time > now() - 2m""", + """SELECT last(value) FROM temp0 WHERE sensor = '28-8B-DC-E9-16-13-01-D5'""", + """SELECT last(value) FROM temp0 WHERE sensor = '28-7F-69-16-17-13-01-A6'""", + ] + }, + { # ################ ################ <- 16 chars + "name": "Stromz\xe1hler " + " Heute ", + "text": "\xf6 Import ____Wh" + "\xf6 Export ____Wh", + "values" : [ + "SELECT last(sum) FROM (SELECT sum(d4_count) FROM stromzaehler3 WHERE time > now() - 1d GROUP BY time(1d) tz('Europe/Berlin'))", + "SELECT last(sum) FROM (SELECT sum(d1_count) FROM stromzaehler3 WHERE time > now() - 1d GROUP BY time(1d) tz('Europe/Berlin'))", + ], + }, + { # ################ ################ <- 16 chars + "name": "Heizung Solar " + " ", + "text": "Von Dach: ____ß " + "Zum Dach: ____ß ", + "values" : [ + "SELECT last(value) FROM temp0 WHERE sensor = '28-7F-69-16-17-13-01-A6'", + "SELECT last(value) FROM temp0 WHERE sensor = '28-99-38-EB-16-13-01-22'", + ] + }, + { # ################ ################ <- 16 chars + "name": "BFT Rheinau " + " ", + "text": "S95-E5: __.___ " + "Diesel: __.___ ", + "values" : [ + """SELECT last(value) FROM "tankerkoenig.SP95-E5" WHERE "name" = 'Rheinau-Freistett - BFT - Freistett'""", + """SELECT last(value) FROM "tankerkoenig.Diesel" WHERE "name" = 'Rheinau-Freistett - BFT - Freistett'""", + ]}, + ] + +def query_one(db, select): + select_esc = select.replace(" ", "%20") + select_esc = select_esc.replace("\"", "%22") + addr = socket.getaddrinfo(HOST, 80)[0][-1] + s = socket.socket() + s.connect(addr) + s.send(b'GET /grafana/api/datasources/proxy/3/query?db={}&q={}&epoch=ms\r\n'.format(db, select_esc)) + s.send(HEADERS) + data = json.load(s) + s.close() + return data['results'][0]['series'][0]['values'][0][1] + +current_display = 0 +def update(): + global UPDATE_RUN, current_display, display, lcd + + lcd.move_to(0, 0) + text = displays[current_display]["text"] + + s = "" + cur = "" + pos = 0 + i = 0 + while i<len(text): + c = text[i] + cur_added = False + if c == '_' or ( len(cur) > 0 and c == '.' ): + cur += c + cur_added = True + + if len(cur) > 0 and (not cur_added or i == len(text) - 1): + value_query = displays[current_display]["values"][pos] + v = query_one("data", value_query) + if isinstance(v, float): + if "." in cur: + precision = len(cur) - cur.index(".") - 1 + else: + precision = 0 + fmt_string = "{{: >{}.{}f}}".format(len(cur), precision) + elif isinstance(v, int): + fmt_string = "{{: >{}d}}".format(len(cur)) + else: + raise Exception("Wrong type: {}".format(type(v))) + s += fmt_string.format(v) + lcd.putstr(s) + s = "" + cur = "" + pos += 1 + + if not cur_added: + s += c + + if not cur_added and i == len(text)-1: + lcd.putstr(s) + s = "" + + i += 1 + + lcd.putstr(s) + +last_update = 0 + +def schedule_update(timer=None, force=False): + global last_update + def ex(none): + try: + update() + except Exception as e: + lcd.clear() + msg = "Error: {}".format(str(e)[0:(32-7)]) + lcd.putstr(msg) + raise e + + now = time.ticks_ms() + if force: + last_update = now + ex(None) + elif now - last_update > 5000: + last_update = now + mp.schedule(ex, None) + +timer = machine.Timer(-1) + +def switch_display(none): + global current_display, displays + + current_display += 1 + current_display %= len(displays) + lcd.move_to(0,0) + lcd.putstr(displays[current_display]['name']) + + # trigger update in 1s + last_update = time.ticks_ms() - 4000 + +last_switch = 0 +def switch_pressed(pin): + global last_switch + now = time.ticks_ms() + if pin.value() == 0 and now - last_switch > 200: + mp.schedule(switch_display, None) + last_switch = now + +switch = machine.Pin(13, machine.Pin.IN) +switch.irq(switch_pressed) + +timer.init(period=500, mode=machine.Timer.PERIODIC, callback=schedule_update) + +print("main.py finished") diff --git a/anzeige0/make.sh b/anzeige0/make.sh new file mode 100755 index 0000000..ef68b45 --- /dev/null +++ b/anzeige0/make.sh @@ -0,0 +1,5 @@ +#!/bin/sh +PY_SOURCES="boot.py main.py .config lcd_api.py nodemcu_gpio_lcd.py" + +. ${0%/*}/../scripts/make.inc.sh + diff --git a/anzeige0/nodemcu_gpio_lcd.py b/anzeige0/nodemcu_gpio_lcd.py new file mode 100644 index 0000000..0bb839c --- /dev/null +++ b/anzeige0/nodemcu_gpio_lcd.py @@ -0,0 +1,168 @@ +"""Implements a HD44780 character LCD connected via NodeMCU GPIO pins.""" + +from lcd_api import LcdApi +from machine import Pin +from utime import sleep_ms, sleep_us + + +class GpioLcd(LcdApi): + """Implements a HD44780 character LCD connected via NodeMCU GPIO pins.""" + + def __init__(self, rs_pin, enable_pin, d0_pin=None, d1_pin=None, + d2_pin=None, d3_pin=None, d4_pin=None, d5_pin=None, + d6_pin=None, d7_pin=None, rw_pin=None, backlight_pin=None, + num_lines=2, num_columns=16): + """Constructs the GpioLcd object. All of the arguments must be Pin + objects which describe which pin the given line from the LCD is + connected to. + + When used in 4-bit mode, only D4, D5, D6, and D7 are physically + connected to the LCD panel. This function allows you call it like + GpioLcd(rs, enable, D4, D5, D6, D7) and it will interpret that as + if you had actually called: + GpioLcd(rs, enable, d4=D4, d5=D5, d6=D6, d7=D7) + + The enable 8-bit mode, you need pass d0 through d7. + + The rw pin isn't used by this library, but if you specify it, then + it will be set low. + """ + self.rs_pin = rs_pin + self.enable_pin = enable_pin + self.rw_pin = rw_pin + self.backlight_pin = backlight_pin + self._4bit = True + if d4_pin and d5_pin and d6_pin and d7_pin: + self.d0_pin = d0_pin + self.d1_pin = d1_pin + self.d2_pin = d2_pin + self.d3_pin = d3_pin + self.d4_pin = d4_pin + self.d5_pin = d5_pin + self.d6_pin = d6_pin + self.d7_pin = d7_pin + if self.d0_pin and self.d1_pin and self.d2_pin and self.d3_pin: + self._4bit = False + else: + # This is really 4-bit mode, and the 4 data pins were just + # passed as the first 4 arguments, so we switch things around. + self.d0_pin = None + self.d1_pin = None + self.d2_pin = None + self.d3_pin = None + self.d4_pin = d0_pin + self.d5_pin = d1_pin + self.d6_pin = d2_pin + self.d7_pin = d3_pin + self.rs_pin.init(Pin.OUT) + self.rs_pin.value(0) + if self.rw_pin: + self.rw_pin.init(Pin.OUT) + self.rw_pin.value(0) + self.enable_pin.init(Pin.OUT) + self.enable_pin.value(0) + self.d4_pin.init(Pin.OUT) + self.d5_pin.init(Pin.OUT) + self.d6_pin.init(Pin.OUT) + self.d7_pin.init(Pin.OUT) + self.d4_pin.value(0) + self.d5_pin.value(0) + self.d6_pin.value(0) + self.d7_pin.value(0) + if not self._4bit: + self.d0_pin.init(Pin.OUT) + self.d1_pin.init(Pin.OUT) + self.d2_pin.init(Pin.OUT) + self.d3_pin.init(Pin.OUT) + self.d0_pin.value(0) + self.d1_pin.value(0) + self.d2_pin.value(0) + self.d3_pin.value(0) + if self.backlight_pin is not None: + self.backlight_pin.init(Pin.OUT) + self.backlight_pin.value(0) + + # See about splitting this into begin + + sleep_ms(20) # Allow LCD time to powerup + # Send reset 3 times + self.hal_write_init_nibble(self.LCD_FUNCTION_RESET) + sleep_ms(5) # need to delay at least 4.1 msec + self.hal_write_init_nibble(self.LCD_FUNCTION_RESET) + sleep_ms(1) + self.hal_write_init_nibble(self.LCD_FUNCTION_RESET) + sleep_ms(1) + cmd = self.LCD_FUNCTION + if not self._4bit: + cmd |= self.LCD_FUNCTION_8BIT + self.hal_write_init_nibble(cmd) + sleep_ms(1) + LcdApi.__init__(self, num_lines, num_columns) + if num_lines > 1: + cmd |= self.LCD_FUNCTION_2LINES + self.hal_write_command(cmd) + + def hal_pulse_enable(self): + """Pulse the enable line high, and then low again.""" + self.enable_pin.value(0) + sleep_us(1) + self.enable_pin.value(1) + sleep_us(1) # Enable pulse needs to be > 450 nsec + self.enable_pin.value(0) + sleep_us(100) # Commands need > 37us to settle + + def hal_write_init_nibble(self, nibble): + """Writes an initialization nibble to the LCD. + + This particular function is only used during initialization. + """ + self.hal_write_4bits(nibble >> 4) + + def hal_backlight_on(self): + """Allows the hal layer to turn the backlight on.""" + if self.backlight_pin: + self.backlight_pin.value(1) + + def hal_backlight_off(self): + """Allows the hal layer to turn the backlight off.""" + if self.backlight_pin: + self.backlight_pin.value(0) + + def hal_write_command(self, cmd): + """Writes a command to the LCD. + + Data is latched on the falling edge of E. + """ + self.rs_pin.value(0) + self.hal_write_8bits(cmd) + if cmd <= 3: + # The home and clear commands require a worst + # case delay of 4.1 msec + sleep_ms(5) + + def hal_write_data(self, data): + """Write data to the LCD.""" + self.rs_pin.value(1) + self.hal_write_8bits(data) + + def hal_write_8bits(self, value): + """Writes 8 bits of data to the LCD.""" + if self.rw_pin: + self.rw_pin.value(0) + if self._4bit: + self.hal_write_4bits(value >> 4) + self.hal_write_4bits(value) + else: + self.d3_pin.value(value & 0x08) + self.d2_pin.value(value & 0x04) + self.d1_pin.value(value & 0x02) + self.d0_pin.value(value & 0x01) + self.hal_write_4bits(value >> 4) + + def hal_write_4bits(self, nibble): + """Writes 4 bits of data to the LCD.""" + self.d7_pin.value(nibble & 0x08) + self.d6_pin.value(nibble & 0x04) + self.d5_pin.value(nibble & 0x02) + self.d4_pin.value(nibble & 0x01) + self.hal_pulse_enable() diff --git a/laermsensor0/Kconfig b/laermsensor0/Kconfig new file mode 100644 index 0000000..afb1c7a --- /dev/null +++ b/laermsensor0/Kconfig @@ -0,0 +1,16 @@ +mainmenu "configuration" + +source ../scripts/Kconfig-wifi + +menu "Influxdb" + +config INFLUXDB_HOST + string "Hostname of influxdb server" + +config INFLUXDB_PORT + int "Port number" + +config INFLUXDB_PATH + string "Path element for POST request" + +endmenu diff --git a/laermsensor0/boot.py b/laermsensor0/boot.py new file mode 100644 index 0000000..0d30893 --- /dev/null +++ b/laermsensor0/boot.py @@ -0,0 +1,2 @@ +config = dict(map(lambda x: (x[0], int(x[1]) if x[1].isdigit() else x[1][1:-1] if x[1].startswith('"') and x[1].endswith('"') else x[1]), map(lambda x: x.strip().split("=", 1), filter(lambda x: '=' in x, open(".config").readlines())))) + diff --git a/laermsensor0/main.py b/laermsensor0/main.py new file mode 100644 index 0000000..b46afdb --- /dev/null +++ b/laermsensor0/main.py @@ -0,0 +1,79 @@ +#!micropython +print('Start main.py') +import socket +from micropython import const, schedule +import network +import machine + +w = network.WLAN() + +print("Waiting for WLAN") +while not w.isconnected(): + machine.idle() + +print("WLAN found") + +HOST = config['CONFIG_INFLUXDB_HOST'] +PORT = config['CONFIG_INFLUXDB_PORT'] +PATH = config['CONFIG_INFLUXDB_PATH'] + +UTF_8 = 'utf8' +HTTP_REQUEST = bytes('POST /%s HTTP/1.1\r\n' % PATH, UTF_8) +HTTP_HOST = bytes('Host: %s\r\n' % HOST, UTF_8) +HTTP_CONTENT_TYPE = bytes('Content-Type: application/x-www-urlencoded\r\n', UTF_8) +HTTP_CONTENT_LENGTH = bytes('Content-Length: %s\r\n', UTF_8) +HTTP_CONNECTION = bytes('Connection: close\r\n', UTF_8) +HTTP_NL = bytes('\r\n', UTF_8) + +def post(data): + print('Send HTTP POST: %s' % data) + addr = socket.getaddrinfo(HOST, PORT)[0][-1] + data = bytes(data, UTF_8) + s = socket.socket() + s.connect(addr) + s.send(HTTP_REQUEST) + s.send(HTTP_HOST) + s.send(HTTP_CONTENT_TYPE) + s.send(HTTP_CONTENT_LENGTH % len(data)) + s.send(HTTP_CONNECTION) + s.send(HTTP_NL) + s.send(data) + + first_data = s.recv(100) + if first_data: + line, *_ = first_data.split(HTTP_NL) + print(line) + while data: + data = s.recv(100) + s.close() + +from machine import Timer, ADC +CONVERT = ( 1024.0 * 3.3 ) / 1000.0 # convert to decibel +values = [] + +def update(args): + (v_max, v_min, v_mean) = args + post('laermsensor0 max=%s,min=%s,mean=%s' % (v_max, v_min, v_mean)) + +def process_adc(t): + global adc + global values + + if len(values) == 50: + v_max = max(values) / CONVERT + v_min = min(values) / CONVERT + v_mean = (sum(values) / 50.0) / CONVERT + schedule(update, (v_max, v_min, v_mean)) + values = [] + else: + values.append(adc.read()) + +adc = ADC(0) +timer = Timer(-1) +timer.init(period=100, mode=Timer.PERIODIC, callback=process_adc) + +def disable(): + global timer + timer.deinit() + +print('main.py finished') diff --git a/laermsensor0/make.sh b/laermsensor0/make.sh new file mode 100755 index 0000000..8266585 --- /dev/null +++ b/laermsensor0/make.sh @@ -0,0 +1,5 @@ +#!/bin/sh +PY_SOURCES="main.py boot.py .config" + +. ${0%/*}/../scripts/make.inc.sh + diff --git a/pinout.jpg b/pinout.jpg Binary files differnew file mode 100644 index 0000000..f7c153a --- /dev/null +++ b/pinout.jpg diff --git a/scripts/Kconfig-wifi b/scripts/Kconfig-wifi new file mode 100644 index 0000000..5864fa1 --- /dev/null +++ b/scripts/Kconfig-wifi @@ -0,0 +1,12 @@ +menu "temp0 configuration" + +config WPA_KEYPHRASE + string "WPA Network secret keyphrase" + +config WPA_SSID + string "WPA SSID" + +config DHCP_HOSTNAME + string "DHCP Hostname" + +endmenu diff --git a/scripts/esp8266-20171101-v1.9.3.bin b/scripts/esp8266-20171101-v1.9.3.bin Binary files differnew file mode 100644 index 0000000..df1adc0 --- /dev/null +++ b/scripts/esp8266-20171101-v1.9.3.bin diff --git a/scripts/esp8266-20180511-v1.9.4.bin b/scripts/esp8266-20180511-v1.9.4.bin Binary files differnew file mode 100644 index 0000000..543f553 --- /dev/null +++ b/scripts/esp8266-20180511-v1.9.4.bin diff --git a/scripts/esptool.py b/scripts/esptool.py new file mode 100755 index 0000000..de7edfb --- /dev/null +++ b/scripts/esptool.py @@ -0,0 +1,2422 @@ +#!/usr/bin/python3 +# +# ESP8266 & ESP32 ROM Bootloader Utility +# Copyright (C) 2014-2016 Fredrik Ahlberg, Angus Gratton, Espressif Systems (Shanghai) PTE LTD, other contributors as noted. +# https://github.com/espressif/esptool +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +# Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import print_function, division + +import argparse +import hashlib +import inspect +import os +import serial +import struct +import sys +import time +import base64 +import zlib +import shlex +import copy +import io + +__version__ = "2.1" + +MAX_UINT32 = 0xffffffff +MAX_UINT24 = 0xffffff + +DEFAULT_TIMEOUT = 3 # timeout for most flash operations +START_FLASH_TIMEOUT = 20 # timeout for starting flash (may perform erase) +CHIP_ERASE_TIMEOUT = 120 # timeout for full chip erase +SYNC_TIMEOUT = 0.1 # timeout for syncing with bootloader + + +DETECTED_FLASH_SIZES = {0x12: '256KB', 0x13: '512KB', 0x14: '1MB', + 0x15: '2MB', 0x16: '4MB', 0x17: '8MB', 0x18: '16MB'} + + +def check_supported_function(func, check_func): + """ + Decorator implementation that wraps a check around an ESPLoader + bootloader function to check if it's supported. + + This is used to capture the multidimensional differences in + functionality between the ESP8266 & ESP32 ROM loaders, and the + software stub that runs on both. Not possible to do this cleanly + via inheritance alone. + """ + def inner(*args, **kwargs): + obj = args[0] + if check_func(obj): + return func(*args, **kwargs) + else: + raise NotImplementedInROMError(obj, func) + return inner + + +def stub_function_only(func): + """ Attribute for a function only supported in the software stub loader """ + return check_supported_function(func, lambda o: o.IS_STUB) + + +def stub_and_esp32_function_only(func): + """ Attribute for a function only supported by software stubs or ESP32 ROM """ + return check_supported_function(func, lambda o: o.IS_STUB or o.CHIP_NAME == "ESP32") + + +PYTHON2 = sys.version_info[0] < 3 # True if on pre-Python 3 + +# Function to return nth byte of a bitstring +# Different behaviour on Python 2 vs 3 +if PYTHON2: + def byte(bitstr, index): + return ord(bitstr[index]) +else: + def byte(bitstr, index): + return bitstr[index] + + +def esp8266_function_only(func): + """ Attribute for a function only supported on ESP8266 """ + return check_supported_function(func, lambda o: o.CHIP_NAME == "ESP8266") + + +class ESPLoader(object): + """ Base class providing access to ESP ROM & softtware stub bootloaders. + Subclasses provide ESP8266 & ESP32 specific functionality. + + Don't instantiate this base class directly, either instantiate a subclass or + call ESPLoader.detect_chip() which will interrogate the chip and return the + appropriate subclass instance. + + """ + CHIP_NAME = "Espressif device" + IS_STUB = False + + DEFAULT_PORT = "/dev/ttyUSB0" + + # Commands supported by ESP8266 ROM bootloader + ESP_FLASH_BEGIN = 0x02 + ESP_FLASH_DATA = 0x03 + ESP_FLASH_END = 0x04 + ESP_MEM_BEGIN = 0x05 + ESP_MEM_END = 0x06 + ESP_MEM_DATA = 0x07 + ESP_SYNC = 0x08 + ESP_WRITE_REG = 0x09 + ESP_READ_REG = 0x0a + + # Some comands supported by ESP32 ROM bootloader (or -8266 w/ stub) + ESP_SPI_SET_PARAMS = 0x0B + ESP_SPI_ATTACH = 0x0D + ESP_CHANGE_BAUDRATE = 0x0F + ESP_FLASH_DEFL_BEGIN = 0x10 + ESP_FLASH_DEFL_DATA = 0x11 + ESP_FLASH_DEFL_END = 0x12 + ESP_SPI_FLASH_MD5 = 0x13 + + # Some commands supported by stub only + ESP_ERASE_FLASH = 0xD0 + ESP_ERASE_REGION = 0xD1 + ESP_READ_FLASH = 0xD2 + ESP_RUN_USER_CODE = 0xD3 + + # Maximum block sized for RAM and Flash writes, respectively. + ESP_RAM_BLOCK = 0x1800 + + FLASH_WRITE_SIZE = 0x400 + + # Default baudrate. The ROM auto-bauds, so we can use more or less whatever we want. + ESP_ROM_BAUD = 115200 + + # First byte of the application image + ESP_IMAGE_MAGIC = 0xe9 + + # Initial state for the checksum routine + ESP_CHECKSUM_MAGIC = 0xef + + # Flash sector size, minimum unit of erase. + FLASH_SECTOR_SIZE = 0x1000 + + UART_DATA_REG_ADDR = 0x60000078 + + # Memory addresses + IROM_MAP_START = 0x40200000 + IROM_MAP_END = 0x40300000 + + # The number of bytes in the UART response that signify command status + STATUS_BYTES_LENGTH = 2 + + def __init__(self, port=DEFAULT_PORT, baud=ESP_ROM_BAUD): + """Base constructor for ESPLoader bootloader interaction + + Don't call this constructor, either instantiate ESP8266ROM + or ESP32ROM, or use ESPLoader.detect_chip(). + + This base class has all of the instance methods for bootloader + functionality supported across various chips & stub + loaders. Subclasses replace the functions they don't support + with ones which throw NotImplementedInROMError(). + + """ + if isinstance(port, serial.Serial): + self._port = port + else: + self._port = serial.serial_for_url(port) + self._slip_reader = slip_reader(self._port) + # setting baud rate in a separate step is a workaround for + # CH341 driver on some Linux versions (this opens at 9600 then + # sets), shouldn't matter for other platforms/drivers. See + # https://github.com/espressif/esptool/issues/44#issuecomment-107094446 + self._set_port_baudrate(baud) + + def _set_port_baudrate(self, baud): + try: + self._port.baudrate = baud + except IOError: + raise FatalError("Failed to set baud rate %d. The driver may not support this rate." % baud) + + @staticmethod + def detect_chip(port=DEFAULT_PORT, baud=ESP_ROM_BAUD, connect_mode='default_reset'): + """ Use serial access to detect the chip type. + + We use the UART's datecode register for this, it's mapped at + the same address on ESP8266 & ESP32 so we can use one + memory read and compare to the datecode register for each chip + type. + + This routine automatically performs ESPLoader.connect() (passing + connect_mode parameter) as part of querying the chip. + """ + detect_port = ESPLoader(port, baud) + detect_port.connect(connect_mode) + print('Detecting chip type...', end='') + sys.stdout.flush() + date_reg = detect_port.read_reg(ESPLoader.UART_DATA_REG_ADDR) + + for cls in [ESP8266ROM, ESP32ROM]: + if date_reg == cls.DATE_REG_VALUE: + # don't connect a second time + inst = cls(detect_port._port, baud) + print(' %s' % inst.CHIP_NAME) + return inst + print('') + raise FatalError("Unexpected UART datecode value 0x%08x. Failed to autodetect chip type." % date_reg) + + """ Read a SLIP packet from the serial port """ + def read(self): + return next(self._slip_reader) + + """ Write bytes to the serial port while performing SLIP escaping """ + def write(self, packet): + buf = b'\xc0' \ + + (packet.replace(b'\xdb',b'\xdb\xdd').replace(b'\xc0',b'\xdb\xdc')) \ + + b'\xc0' + self._port.write(buf) + + """ Calculate checksum of a blob, as it is defined by the ROM """ + @staticmethod + def checksum(data, state=ESP_CHECKSUM_MAGIC): + for b in data: + if type(b) is int: # python 2/3 compat + state ^= b + else: + state ^= ord(b) + + return state + + """ Send a request and read the response """ + def command(self, op=None, data=b"", chk=0, wait_response=True): + if op is not None: + pkt = struct.pack(b'<BBHI', 0x00, op, len(data), chk) + data + self.write(pkt) + + if not wait_response: + return + + # tries to get a response until that response has the + # same operation as the request or a retries limit has + # exceeded. This is needed for some esp8266s that + # reply with more sync responses than expected. + for retry in range(100): + p = self.read() + if len(p) < 8: + continue + (resp, op_ret, len_ret, val) = struct.unpack('<BBHI', p[:8]) + if resp != 1: + continue + data = p[8:] + if op is None or op_ret == op: + return val, data + + raise FatalError("Response doesn't match request") + + def check_command(self, op_description, op=None, data=b'', chk=0): + """ + Execute a command with 'command', check the result code and throw an appropriate + FatalError if it fails. + + Returns the "result" of a successful command. + """ + val, data = self.command(op, data, chk) + + # things are a bit weird here, bear with us + + # the status bytes are the last 2/4 bytes in the data (depending on chip) + if len(data) < self.STATUS_BYTES_LENGTH: + raise FatalError("Failed to %s. Only got %d byte status response." % (op_description, len(data))) + status_bytes = data[-self.STATUS_BYTES_LENGTH:] + # we only care if the first one is non-zero. If it is, the second byte is a reason. + if byte(status_bytes, 0) != 0: + raise FatalError.WithResult('Failed to %s' % op_description, status_bytes) + + # if we had more data than just the status bytes, return it as the result + # (this is used by the md5sum command, maybe other commands?) + if len(data) > self.STATUS_BYTES_LENGTH: + return data[:-self.STATUS_BYTES_LENGTH] + else: # otherwise, just return the 'val' field which comes from the reply header (this is used by read_reg) + return val + + def flush_input(self): + self._port.flushInput() + self._slip_reader = slip_reader(self._port) + + def sync(self): + self.command(self.ESP_SYNC, b'\x07\x07\x12\x20' + 32 * b'\x55') + for i in range(7): + self.command() + + def _connect_attempt(self, mode='default_reset', esp32r0_delay=False): + """ A single connection attempt, with esp32r0 workaround options """ + # esp32r0_delay is a workaround for bugs with the most common auto reset + # circuit and Windows, if the EN pin on the dev board does not have + # enough capacitance. + # + # Newer dev boards shouldn't have this problem (higher value capacitor + # on the EN pin), and ESP32 revision 1 can't use this workaround as it + # relies on a silicon bug. + # + # Details: https://github.com/espressif/esptool/issues/136 + last_error = None + + # issue reset-to-bootloader: + # RTS = either CH_PD/EN or nRESET (both active low = chip in reset + # DTR = GPIO0 (active low = boot to flasher) + # + # DTR & RTS are active low signals, + # ie True = pin @ 0V, False = pin @ VCC. + if mode != 'no_reset': + self._port.setDTR(False) # IO0=HIGH + self._port.setRTS(True) # EN=LOW, chip in reset + time.sleep(0.1) + if esp32r0_delay: + # Some chips are more likely to trigger the esp32r0 + # watchdog reset silicon bug if they're held with EN=LOW + # for a longer period + time.sleep(1.2) + self._port.setDTR(True) # IO0=LOW + self._port.setRTS(False) # EN=HIGH, chip out of reset + if esp32r0_delay: + # Sleep longer after reset. + # This workaround only works on revision 0 ESP32 chips, + # it exploits a silicon bug spurious watchdog reset. + time.sleep(0.4) # allow watchdog reset to occur + time.sleep(0.05) + self._port.setDTR(False) # IO0=HIGH, done + + self._port.timeout = SYNC_TIMEOUT + for _ in range(5): + try: + self.flush_input() + self._port.flushOutput() + self.sync() + self._port.timeout = DEFAULT_TIMEOUT + return None + except FatalError as e: + if esp32r0_delay: + print('_', end='') + else: + print('.', end='') + sys.stdout.flush() + time.sleep(0.05) + last_error = e + return last_error + + def connect(self, mode='default_reset'): + """ Try connecting repeatedly until successful, or giving up """ + print('Connecting...', end='') + sys.stdout.flush() + last_error = None + + try: + for _ in range(10): + last_error = self._connect_attempt(mode=mode, esp32r0_delay=False) + if last_error is None: + return + last_error = self._connect_attempt(mode=mode, esp32r0_delay=True) + if last_error is None: + return + finally: + print('') # end 'Connecting...' line + raise FatalError('Failed to connect to %s: %s' % (self.CHIP_NAME, last_error)) + + """ Read memory address in target """ + def read_reg(self, addr): + # we don't call check_command here because read_reg() function is called + # when detecting chip type, and the way we check for success (STATUS_BYTES_LENGTH) is different + # for different chip types (!) + val, data = self.command(self.ESP_READ_REG, struct.pack('<I', addr)) + if byte(data, 0) != 0: + raise FatalError.WithResult("Failed to read register address %08x" % addr, data) + return val + + """ Write to memory address in target """ + def write_reg(self, addr, value, mask=0xFFFFFFFF, delay_us=0): + return self.check_command("write target memory", self.ESP_WRITE_REG, + struct.pack('<IIII', addr, value, mask, delay_us)) + + """ Start downloading an application image to RAM """ + def mem_begin(self, size, blocks, blocksize, offset): + return self.check_command("enter RAM download mode", self.ESP_MEM_BEGIN, + struct.pack('<IIII', size, blocks, blocksize, offset)) + + """ Send a block of an image to RAM """ + def mem_block(self, data, seq): + return self.check_command("write to target RAM", self.ESP_MEM_DATA, + struct.pack('<IIII', len(data), seq, 0, 0) + data, + self.checksum(data)) + + """ Leave download mode and run the application """ + def mem_finish(self, entrypoint=0): + return self.check_command("leave RAM download mode", self.ESP_MEM_END, + struct.pack('<II', int(entrypoint == 0), entrypoint)) + + """ Start downloading to Flash (performs an erase) + + Returns number of blocks (of size self.FLASH_WRITE_SIZE) to write. + """ + def flash_begin(self, size, offset): + num_blocks = (size + self.FLASH_WRITE_SIZE - 1) // self.FLASH_WRITE_SIZE + erase_size = self.get_erase_size(offset, size) + + self._port.timeout = START_FLASH_TIMEOUT + t = time.time() + self.check_command("enter Flash download mode", self.ESP_FLASH_BEGIN, + struct.pack('<IIII', erase_size, num_blocks, self.FLASH_WRITE_SIZE, offset)) + if size != 0 and not self.IS_STUB: + print("Took %.2fs to erase flash block" % (time.time() - t)) + self._port.timeout = DEFAULT_TIMEOUT + return num_blocks + + """ Write block to flash """ + def flash_block(self, data, seq): + self.check_command("write to target Flash after seq %d" % seq, + self.ESP_FLASH_DATA, + struct.pack('<IIII', len(data), seq, 0, 0) + data, + self.checksum(data)) + + """ Leave flash mode and run/reboot """ + def flash_finish(self, reboot=False): + pkt = struct.pack('<I', int(not reboot)) + # stub sends a reply to this command + self.check_command("leave Flash mode", self.ESP_FLASH_END, pkt) + + """ Run application code in flash """ + def run(self, reboot=False): + # Fake flash begin immediately followed by flash end + self.flash_begin(0, 0) + self.flash_finish(reboot) + + """ Read SPI flash manufacturer and device id """ + def flash_id(self): + SPIFLASH_RDID = 0x9F + return self.run_spiflash_command(SPIFLASH_RDID, b"", 24) + + def parse_flash_size_arg(self, arg): + try: + return self.FLASH_SIZES[arg] + except KeyError: + raise FatalError("Flash size '%s' is not supported by this chip type. Supported sizes: %s" + % (arg, ", ".join(self.FLASH_SIZES.keys()))) + + def run_stub(self, stub=None): + if stub is None: + if self.IS_STUB: + raise FatalError("Not possible for a stub to load another stub (memory likely to overlap.)") + stub = self.STUB_CODE + + # Upload + print("Uploading stub...") + for field in ['text', 'data']: + if field in stub: + offs = stub[field + "_start"] + length = len(stub[field]) + blocks = (length + self.ESP_RAM_BLOCK - 1) // self.ESP_RAM_BLOCK + self.mem_begin(length, blocks, self.ESP_RAM_BLOCK, offs) + for seq in range(blocks): + from_offs = seq * self.ESP_RAM_BLOCK + to_offs = from_offs + self.ESP_RAM_BLOCK + self.mem_block(stub[field][from_offs:to_offs], seq) + print("Running stub...") + self.mem_finish(stub['entry']) + + p = self.read() + if p != b'OHAI': + raise FatalError("Failed to start stub. Unexpected response: %s" % p) + print("Stub running...") + return self.STUB_CLASS(self) + + @stub_and_esp32_function_only + def flash_defl_begin(self, size, compsize, offset): + """ Start downloading compressed data to Flash (performs an erase) + + Returns number of blocks (size self.FLASH_WRITE_SIZE) to write. + """ + num_blocks = (compsize + self.FLASH_WRITE_SIZE - 1) // self.FLASH_WRITE_SIZE + erase_blocks = (size + self.FLASH_WRITE_SIZE - 1) // self.FLASH_WRITE_SIZE + + self._port.timeout = START_FLASH_TIMEOUT + t = time.time() + if self.IS_STUB: + write_size = size # stub expects number of bytes here, manages erasing internally + else: + write_size = erase_blocks * self.FLASH_WRITE_SIZE # ROM expects rounded up to erase block size + print("Compressed %d bytes to %d..." % (size, compsize)) + self.check_command("enter compressed flash mode", self.ESP_FLASH_DEFL_BEGIN, + struct.pack('<IIII', write_size, num_blocks, self.FLASH_WRITE_SIZE, offset)) + if size != 0 and not self.IS_STUB: + # (stub erases as it writes, but ROM loaders erase on begin) + print("Took %.2fs to erase flash block" % (time.time() - t)) + self._port.timeout = DEFAULT_TIMEOUT + return num_blocks + + """ Write block to flash, send compressed """ + @stub_and_esp32_function_only + def flash_defl_block(self, data, seq): + self.check_command("write compressed data to flash after seq %d" % seq, + self.ESP_FLASH_DEFL_DATA, struct.pack('<IIII', len(data), seq, 0, 0) + data, self.checksum(data)) + + """ Leave compressed flash mode and run/reboot """ + @stub_and_esp32_function_only + def flash_defl_finish(self, reboot=False): + if not reboot and not self.IS_STUB: + # skip sending flash_finish to ROM loader, as this + # exits the bootloader. Stub doesn't do this. + return + pkt = struct.pack('<I', int(not reboot)) + self.check_command("leave compressed flash mode", self.ESP_FLASH_DEFL_END, pkt) + self.in_bootloader = False + + @stub_and_esp32_function_only + def flash_md5sum(self, addr, size): + # the MD5 command returns additional bytes in the standard + # command reply slot + res = self.check_command('calculate md5sum', self.ESP_SPI_FLASH_MD5, struct.pack('<IIII', addr, size, 0, 0)) + + if len(res) == 32: + return res.decode("utf-8") # already hex formatted + elif len(res) == 16: + return hexify(res).lower() + else: + raise FatalError("MD5Sum command returned unexpected result: %r" % res) + + @stub_and_esp32_function_only + def change_baud(self, baud): + print("Changing baud rate to %d" % baud) + self.command(self.ESP_CHANGE_BAUDRATE, struct.pack('<II', baud, 0)) + print("Changed.") + self._set_port_baudrate(baud) + time.sleep(0.05) # get rid of crap sent during baud rate change + self.flush_input() + + @stub_function_only + def erase_flash(self): + # depending on flash chip model the erase may take this long (maybe longer!) + self._port.timeout = CHIP_ERASE_TIMEOUT + try: + self.check_command("erase flash", self.ESP_ERASE_FLASH) + finally: + self._port.timeout = DEFAULT_TIMEOUT + + @stub_function_only + def erase_region(self, offset, size): + if offset % self.FLASH_SECTOR_SIZE != 0: + raise FatalError("Offset to erase from must be a multiple of 4096") + if size % self.FLASH_SECTOR_SIZE != 0: + raise FatalError("Size of data to erase must be a multiple of 4096") + self.check_command("erase region", self.ESP_ERASE_REGION, struct.pack('<II', offset, size)) + + @stub_function_only + def read_flash(self, offset, length, progress_fn=None): + # issue a standard bootloader command to trigger the read + self.check_command("read flash", self.ESP_READ_FLASH, + struct.pack('<IIII', + offset, + length, + self.FLASH_SECTOR_SIZE, + 64)) + # now we expect (length // block_size) SLIP frames with the data + data = b'' + while len(data) < length: + p = self.read() + data += p + self.write(struct.pack('<I', len(data))) + if progress_fn and (len(data) % 1024 == 0 or len(data) == length): + progress_fn(len(data), length) + if progress_fn: + progress_fn(len(data), length) + if len(data) > length: + raise FatalError('Read more than expected') + digest_frame = self.read() + if len(digest_frame) != 16: + raise FatalError('Expected digest, got: %s' % hexify(digest_frame)) + expected_digest = hexify(digest_frame).upper() + digest = hashlib.md5(data).hexdigest().upper() + if digest != expected_digest: + raise FatalError('Digest mismatch: expected %s, got %s' % (expected_digest, digest)) + return data + + def flash_spi_attach(self, hspi_arg): + """Send SPI attach command to enable the SPI flash pins + + ESP8266 ROM does this when you send flash_begin, ESP32 ROM + has it as a SPI command. + """ + # last 3 bytes in ESP_SPI_ATTACH argument are reserved values + arg = struct.pack('<I', hspi_arg) + if not self.IS_STUB: + # ESP32 ROM loader takes additional 'is legacy' arg, which is not + # currently supported in the stub loader or esptool.py (as it's not usually needed.) + is_legacy = 0 + arg += struct.pack('BBBB', is_legacy, 0, 0, 0) + self.check_command("configure SPI flash pins", ESP32ROM.ESP_SPI_ATTACH, arg) + + def flash_set_parameters(self, size): + """Tell the ESP bootloader the parameters of the chip + + Corresponds to the "flashchip" data structure that the ROM + has in RAM. + + 'size' is in bytes. + + All other flash parameters are currently hardcoded (on ESP8266 + these are mostly ignored by ROM code, on ESP32 I'm not sure.) + """ + fl_id = 0 + total_size = size + block_size = 64 * 1024 + sector_size = 4 * 1024 + page_size = 256 + status_mask = 0xffff + self.check_command("set SPI params", ESP32ROM.ESP_SPI_SET_PARAMS, + struct.pack('<IIIIII', fl_id, total_size, block_size, sector_size, page_size, status_mask)) + + def run_spiflash_command(self, spiflash_command, data=b"", read_bits=0): + """Run an arbitrary SPI flash command. + + This function uses the "USR_COMMAND" functionality in the ESP + SPI hardware, rather than the precanned commands supported by + hardware. So the value of spiflash_command is an actual command + byte, sent over the wire. + + After writing command byte, writes 'data' to MOSI and then + reads back 'read_bits' of reply on MISO. Result is a number. + """ + + # SPI_USR register flags + SPI_USR_COMMAND = (1 << 31) + SPI_USR_MISO = (1 << 28) + SPI_USR_MOSI = (1 << 27) + + # SPI registers, base address differs ESP32 vs 8266 + base = self.SPI_REG_BASE + SPI_CMD_REG = base + 0x00 + SPI_USR_REG = base + 0x1C + SPI_USR1_REG = base + 0x20 + SPI_USR2_REG = base + 0x24 + SPI_W0_REG = base + self.SPI_W0_OFFS + + # following two registers are ESP32 only + if self.SPI_HAS_MOSI_DLEN_REG: + # ESP32 has a more sophisticated wayto set up "user" commands + def set_data_lengths(mosi_bits, miso_bits): + SPI_MOSI_DLEN_REG = base + 0x28 + SPI_MISO_DLEN_REG = base + 0x2C + if mosi_bits > 0: + self.write_reg(SPI_MOSI_DLEN_REG, mosi_bits - 1) + if miso_bits > 0: + self.write_reg(SPI_MISO_DLEN_REG, miso_bits - 1) + else: + + def set_data_lengths(mosi_bits, miso_bits): + SPI_DATA_LEN_REG = SPI_USR1_REG + SPI_MOSI_BITLEN_S = 17 + SPI_MISO_BITLEN_S = 8 + mosi_mask = 0 if (mosi_bits == 0) else (mosi_bits - 1) + miso_mask = 0 if (miso_bits == 0) else (miso_bits - 1) + self.write_reg(SPI_DATA_LEN_REG, + (miso_mask << SPI_MISO_BITLEN_S) | ( + mosi_mask << SPI_MOSI_BITLEN_S)) + + # SPI peripheral "command" bitmasks for SPI_CMD_REG + SPI_CMD_USR = (1 << 18) + + # shift values + SPI_USR2_DLEN_SHIFT = 28 + + if read_bits > 32: + raise FatalError("Reading more than 32 bits back from a SPI flash operation is unsupported") + if len(data) > 64: + raise FatalError("Writing more than 64 bytes of data with one SPI command is unsupported") + + data_bits = len(data) * 8 + old_spi_usr = self.read_reg(SPI_USR_REG) + old_spi_usr2 = self.read_reg(SPI_USR2_REG) + flags = SPI_USR_COMMAND + if read_bits > 0: + flags |= SPI_USR_MISO + if data_bits > 0: + flags |= SPI_USR_MOSI + set_data_lengths(data_bits, read_bits) + self.write_reg(SPI_USR_REG, flags) + self.write_reg(SPI_USR2_REG, + (7 << SPI_USR2_DLEN_SHIFT) | spiflash_command) + if data_bits == 0: + self.write_reg(SPI_W0_REG, 0) # clear data register before we read it + else: + data = pad_to(data, 4, b'\00') # pad to 32-bit multiple + words = struct.unpack("I" * (len(data) // 4), data) + next_reg = SPI_W0_REG + for word in words: + self.write_reg(next_reg, word) + next_reg += 4 + self.write_reg(SPI_CMD_REG, SPI_CMD_USR) + + def wait_done(): + for _ in range(10): + if (self.read_reg(SPI_CMD_REG) & SPI_CMD_USR) == 0: + return + raise FatalError("SPI command did not complete in time") + wait_done() + + status = self.read_reg(SPI_W0_REG) + # restore some SPI controller registers + self.write_reg(SPI_USR_REG, old_spi_usr) + self.write_reg(SPI_USR2_REG, old_spi_usr2) + return status + + def read_status(self, num_bytes=2): + """Read up to 24 bits (num_bytes) of SPI flash status register contents + via RDSR, RDSR2, RDSR3 commands + + Not all SPI flash supports all three commands. The upper 1 or 2 + bytes may be 0xFF. + """ + SPIFLASH_RDSR = 0x05 + SPIFLASH_RDSR2 = 0x35 + SPIFLASH_RDSR3 = 0x15 + + status = 0 + shift = 0 + for cmd in [SPIFLASH_RDSR, SPIFLASH_RDSR2, SPIFLASH_RDSR3][0:num_bytes]: + status += self.run_spiflash_command(cmd, read_bits=8) << shift + shift += 8 + return status + + def write_status(self, new_status, num_bytes=2, set_non_volatile=False): + """Write up to 24 bits (num_bytes) of new status register + + num_bytes can be 1, 2 or 3. + + Not all flash supports the additional commands to write the + second and third byte of the status register. When writing 2 + bytes, esptool also sends a 16-byte WRSR command (as some + flash types use this instead of WRSR2.) + + If the set_non_volatile flag is set, non-volatile bits will + be set as well as volatile ones (WREN used instead of WEVSR). + + """ + SPIFLASH_WRSR = 0x01 + SPIFLASH_WRSR2 = 0x31 + SPIFLASH_WRSR3 = 0x11 + SPIFLASH_WEVSR = 0x50 + SPIFLASH_WREN = 0x06 + SPIFLASH_WRDI = 0x04 + + enable_cmd = SPIFLASH_WREN if set_non_volatile else SPIFLASH_WEVSR + + # try using a 16-bit WRSR (not supported by all chips) + # this may be redundant, but shouldn't hurt + if num_bytes == 2: + self.run_spiflash_command(enable_cmd) + self.run_spiflash_command(SPIFLASH_WRSR, struct.pack("<H", new_status)) + + # also try using individual commands (also not supported by all chips for num_bytes 2 & 3) + for cmd in [SPIFLASH_WRSR, SPIFLASH_WRSR2, SPIFLASH_WRSR3][0:num_bytes]: + self.run_spiflash_command(enable_cmd) + self.run_spiflash_command(cmd, struct.pack("B", new_status & 0xFF)) + new_status >>= 8 + + self.run_spiflash_command(SPIFLASH_WRDI) + + def hard_reset(self): + self._port.setRTS(True) # EN->LOW + time.sleep(0.1) + self._port.setRTS(False) + + def soft_reset(self, stay_in_bootloader): + if not self.IS_STUB: + if stay_in_bootloader: + return # ROM bootloader is already in bootloader! + else: + # 'run user code' is as close to a soft reset as we can do + self.flash_begin(0, 0) + self.flash_finish(False) + else: + if stay_in_bootloader: + # soft resetting from the stub loader + # will re-load the ROM bootloader + self.flash_begin(0, 0) + self.flash_finish(True) + elif self.CHIP_NAME != "ESP8266": + raise FatalError("Soft resetting is currently only supported on ESP8266") + else: + # running user code from stub loader requires some hacks + # in the stub loader + self.command(self.ESP_RUN_USER_CODE, wait_response=False) + + +class ESP8266ROM(ESPLoader): + """ Access class for ESP8266 ROM bootloader + """ + CHIP_NAME = "ESP8266" + IS_STUB = False + + DATE_REG_VALUE = 0x00062000 + + # OTP ROM addresses + ESP_OTP_MAC0 = 0x3ff00050 + ESP_OTP_MAC1 = 0x3ff00054 + ESP_OTP_MAC3 = 0x3ff0005c + + SPI_REG_BASE = 0x60000200 + SPI_W0_OFFS = 0x40 + SPI_HAS_MOSI_DLEN_REG = False + + FLASH_SIZES = { + '512KB':0x00, + '256KB':0x10, + '1MB':0x20, + '2MB':0x30, + '4MB':0x40, + '2MB-c1': 0x50, + '4MB-c1':0x60, + '8MB':0x80, + '16MB':0x90, + } + + BOOTLOADER_FLASH_OFFSET = 0 + + def get_chip_description(self): + return "ESP8266" + + def flash_spi_attach(self, hspi_arg): + if self.IS_STUB: + super(ESP8266ROM, self).flash_spi_attach(hspi_arg) + else: + # ESP8266 ROM has no flash_spi_attach command in serial protocol, + # but flash_begin will do it + self.flash_begin(0, 0) + + def flash_set_parameters(self, size): + # not implemented in ROM, but OK to silently skip for ROM + if self.IS_STUB: + super(ESP8266ROM, self).flash_set_parameters(size) + + def chip_id(self): + """ Read Chip ID from OTP ROM - see http://esp8266-re.foogod.com/wiki/System_get_chip_id_%28IoT_RTOS_SDK_0.9.9%29 """ + id0 = self.read_reg(self.ESP_OTP_MAC0) + id1 = self.read_reg(self.ESP_OTP_MAC1) + return (id0 >> 24) | ((id1 & MAX_UINT24) << 8) + + def read_mac(self): + """ Read MAC from OTP ROM """ + mac0 = self.read_reg(self.ESP_OTP_MAC0) + mac1 = self.read_reg(self.ESP_OTP_MAC1) + mac3 = self.read_reg(self.ESP_OTP_MAC3) + if (mac3 != 0): + oui = ((mac3 >> 16) & 0xff, (mac3 >> 8) & 0xff, mac3 & 0xff) + elif ((mac1 >> 16) & 0xff) == 0: + oui = (0x18, 0xfe, 0x34) + elif ((mac1 >> 16) & 0xff) == 1: + oui = (0xac, 0xd0, 0x74) + else: + raise FatalError("Unknown OUI") + return oui + ((mac1 >> 8) & 0xff, mac1 & 0xff, (mac0 >> 24) & 0xff) + + def get_erase_size(self, offset, size): + """ Calculate an erase size given a specific size in bytes. + + Provides a workaround for the bootloader erase bug.""" + + sectors_per_block = 16 + sector_size = self.FLASH_SECTOR_SIZE + num_sectors = (size + sector_size - 1) // sector_size + start_sector = offset // sector_size + + head_sectors = sectors_per_block - (start_sector % sectors_per_block) + if num_sectors < head_sectors: + head_sectors = num_sectors + + if num_sectors < 2 * head_sectors: + return (num_sectors + 1) // 2 * sector_size + else: + return (num_sectors - head_sectors) * sector_size + + +class ESP8266StubLoader(ESP8266ROM): + """ Access class for ESP8266 stub loader, runs on top of ROM. + """ + FLASH_WRITE_SIZE = 0x4000 # matches MAX_WRITE_BLOCK in stub_loader.c + IS_STUB = True + + def __init__(self, rom_loader): + self._port = rom_loader._port + self.flush_input() # resets _slip_reader + + def get_erase_size(self, offset, size): + return size # stub doesn't have same size bug as ROM loader + + +ESP8266ROM.STUB_CLASS = ESP8266StubLoader + + +class ESP32ROM(ESPLoader): + """Access class for ESP32 ROM bootloader + + """ + CHIP_NAME = "ESP32" + IS_STUB = False + + DATE_REG_VALUE = 0x15122500 + + IROM_MAP_START = 0x400d0000 + IROM_MAP_END = 0x40400000 + DROM_MAP_START = 0x3F400000 + DROM_MAP_END = 0x3F800000 + + # ESP32 uses a 4 byte status reply + STATUS_BYTES_LENGTH = 4 + + SPI_REG_BASE = 0x60002000 + EFUSE_REG_BASE = 0x6001a000 + + SPI_W0_OFFS = 0x80 + SPI_HAS_MOSI_DLEN_REG = True + + FLASH_SIZES = { + '1MB':0x00, + '2MB':0x10, + '4MB':0x20, + '8MB':0x30, + '16MB':0x40 + } + + BOOTLOADER_FLASH_OFFSET = 0x1000 + + def get_chip_description(self): + blk3 = self.read_efuse(3) + chip_version = (blk3 >> 12) & 0xF + pkg_version = (blk3 >> 9) & 0x07 + + silicon_rev = { + 0: "0", + 8: "1" + }.get(chip_version, "(unknown 0x%x)" % chip_version) + + chip_name = { + 0: "ESP32D0WDQ6", + 1: "ESP32D0WDQ5", + 2: "ESP32D2WDQ5", + }.get(pkg_version, "unknown ESP32") + + return "%s (revision %s)" % (chip_name, silicon_rev) + + def read_efuse(self, n): + """ Read the nth word of the ESP3x EFUSE region. """ + return self.read_reg(self.EFUSE_REG_BASE + (4 * n)) + + def chip_id(self): + word16 = self.read_efuse(1) + word17 = self.read_efuse(2) + return ((word17 & MAX_UINT24) << 24) | (word16 >> 8) & MAX_UINT24 + + def read_mac(self): + """ Read MAC from EFUSE region """ + words = [self.read_efuse(2), self.read_efuse(1)] + bitstring = struct.pack(">II", *words) + bitstring = bitstring[2:8] # trim the 2 byte CRC + try: + return tuple(ord(b) for b in bitstring) + except TypeError: # Python 3, bitstring elements are already bytes + return tuple(bitstring) + + def get_erase_size(self, offset, size): + return size + + +class ESP32StubLoader(ESP32ROM): + """ Access class for ESP32 stub loader, runs on top of ROM. + """ + FLASH_WRITE_SIZE = 0x4000 # matches MAX_WRITE_BLOCK in stub_loader.c + STATUS_BYTES_LENGTH = 2 # same as ESP8266, different to ESP32 ROM + IS_STUB = True + + def __init__(self, rom_loader): + self._port = rom_loader._port + self.flush_input() # resets _slip_reader + + +ESP32ROM.STUB_CLASS = ESP32StubLoader + + +class ESPBOOTLOADER(object): + """ These are constants related to software ESP bootloader, working with 'v2' image files """ + + # First byte of the "v2" application image + IMAGE_V2_MAGIC = 0xea + + # First 'segment' value in a "v2" application image, appears to be a constant version value? + IMAGE_V2_SEGMENT = 4 + + +def LoadFirmwareImage(chip, filename): + """ Load a firmware image. Can be for ESP8266 or ESP32. ESP8266 images will be examined to determine if they are + original ROM firmware images (ESPFirmwareImage) or "v2" OTA bootloader images. + + Returns a BaseFirmwareImage subclass, either ESPFirmwareImage (v1) or OTAFirmwareImage (v2). + """ + with open(filename, 'rb') as f: + if chip == 'esp32': + return ESP32FirmwareImage(f) + else: # Otherwise, ESP8266 so look at magic to determine the image type + magic = ord(f.read(1)) + f.seek(0) + if magic == ESPLoader.ESP_IMAGE_MAGIC: + return ESPFirmwareImage(f) + elif magic == ESPBOOTLOADER.IMAGE_V2_MAGIC: + return OTAFirmwareImage(f) + else: + raise FatalError("Invalid image magic number: %d" % magic) + + +class ImageSegment(object): + """ Wrapper class for a segment in an ESP image + (very similar to a section in an ELFImage also) """ + def __init__(self, addr, data, file_offs=None): + self.addr = addr + # pad all ImageSegments to at least 4 bytes length + self.data = pad_to(data, 4, b'\x00') + self.file_offs = file_offs + self.include_in_checksum = True + + def copy_with_new_addr(self, new_addr): + """ Return a new ImageSegment with same data, but mapped at + a new address. """ + return ImageSegment(new_addr, self.data, 0) + + def split_image(self, split_len): + """ Return a new ImageSegment which splits "split_len" bytes + from the beginning of the data. Remaining bytes are kept in + this segment object (and the start address is adjusted to match.) """ + result = copy.copy(self) + result.data = self.data[:split_len] + self.data = self.data[split_len:] + self.addr += split_len + self.file_offs = None + result.file_offs = None + return result + + def __repr__(self): + r = "len 0x%05x load 0x%08x" % (len(self.data), self.addr) + if self.file_offs is not None: + r += " file_offs 0x%08x" % (self.file_offs) + return r + + +class ELFSection(ImageSegment): + """ Wrapper class for a section in an ELF image, has a section + name as well as the common properties of an ImageSegment. """ + def __init__(self, name, addr, data): + super(ELFSection, self).__init__(addr, data) + self.name = name.decode("utf-8") + + def __repr__(self): + return "%s %s" % (self.name, super(ELFSection, self).__repr__()) + + +class BaseFirmwareImage(object): + SEG_HEADER_LEN = 8 + + """ Base class with common firmware image functions """ + def __init__(self): + self.segments = [] + self.entrypoint = 0 + + def load_common_header(self, load_file, expected_magic): + (magic, segments, self.flash_mode, self.flash_size_freq, self.entrypoint) = struct.unpack('<BBBBI', load_file.read(8)) + + if magic != expected_magic or segments > 16: + raise FatalError('Invalid firmware image magic=%d segments=%d' % (magic, segments)) + return segments + + def load_segment(self, f, is_irom_segment=False): + """ Load the next segment from the image file """ + file_offs = f.tell() + (offset, size) = struct.unpack('<II', f.read(8)) + self.warn_if_unusual_segment(offset, size, is_irom_segment) + segment_data = f.read(size) + if len(segment_data) < size: + raise FatalError('End of file reading segment 0x%x, length %d (actual length %d)' % (offset, size, len(segment_data))) + segment = ImageSegment(offset, segment_data, file_offs) + self.segments.append(segment) + return segment + + def warn_if_unusual_segment(self, offset, size, is_irom_segment): + if not is_irom_segment: + if offset > 0x40200000 or offset < 0x3ffe0000 or size > 65536: + print('WARNING: Suspicious segment 0x%x, length %d' % (offset, size)) + + def save_segment(self, f, segment, checksum=None): + """ Save the next segment to the image file, return next checksum value if provided """ + f.write(struct.pack('<II', segment.addr, len(segment.data))) + f.write(segment.data) + if checksum is not None: + return ESPLoader.checksum(segment.data, checksum) + + def read_checksum(self, f): + """ Return ESPLoader checksum from end of just-read image """ + # Skip the padding. The checksum is stored in the last byte so that the + # file is a multiple of 16 bytes. + align_file_position(f, 16) + return ord(f.read(1)) + + def calculate_checksum(self): + """ Calculate checksum of loaded image, based on segments in + segment array. + """ + checksum = ESPLoader.ESP_CHECKSUM_MAGIC + for seg in self.segments: + if seg.include_in_checksum: + checksum = ESPLoader.checksum(seg.data, checksum) + return checksum + + def append_checksum(self, f, checksum): + """ Append ESPLoader checksum to the just-written image """ + align_file_position(f, 16) + f.write(struct.pack(b'B', checksum)) + + def write_common_header(self, f, segments): + f.write(struct.pack('<BBBBI', ESPLoader.ESP_IMAGE_MAGIC, len(segments), + self.flash_mode, self.flash_size_freq, self.entrypoint)) + + def is_irom_addr(self, addr): + """ Returns True if an address starts in the irom region. + Valid for ESP8266 only. + """ + return ESP8266ROM.IROM_MAP_START <= addr < ESP8266ROM.IROM_MAP_END + + def get_irom_segment(self): + irom_segments = [s for s in self.segments if self.is_irom_addr(s.addr)] + if len(irom_segments) > 0: + if len(irom_segments) != 1: + raise FatalError('Found %d segments that could be irom0. Bad ELF file?' % len(irom_segments)) + return irom_segments[0] + return None + + def get_non_irom_segments(self): + irom_segment = self.get_irom_segment() + return [s for s in self.segments if s != irom_segment] + + +class ESPFirmwareImage(BaseFirmwareImage): + """ 'Version 1' firmware image, segments loaded directly by the ROM bootloader. """ + + ROM_LOADER = ESP8266ROM + + def __init__(self, load_file=None): + super(ESPFirmwareImage, self).__init__() + self.flash_mode = 0 + self.flash_size_freq = 0 + self.version = 1 + + if load_file is not None: + segments = self.load_common_header(load_file, ESPLoader.ESP_IMAGE_MAGIC) + + for _ in range(segments): + self.load_segment(load_file) + self.checksum = self.read_checksum(load_file) + + def default_output_name(self, input_file): + """ Derive a default output name from the ELF name. """ + return input_file + '-' + + def save(self, basename): + """ Save a set of V1 images for flashing. Parameter is a base filename. """ + # IROM data goes in its own plain binary file + irom_segment = self.get_irom_segment() + if irom_segment is not None: + with open("%s0x%05x.bin" % (basename, irom_segment.addr - ESP8266ROM.IROM_MAP_START), "wb") as f: + f.write(irom_segment.data) + + # everything but IROM goes at 0x00000 in an image file + normal_segments = self.get_non_irom_segments() + with open("%s0x00000.bin" % basename, 'wb') as f: + self.write_common_header(f, normal_segments) + checksum = ESPLoader.ESP_CHECKSUM_MAGIC + for segment in normal_segments: + checksum = self.save_segment(f, segment, checksum) + self.append_checksum(f, checksum) + + +class OTAFirmwareImage(BaseFirmwareImage): + """ 'Version 2' firmware image, segments loaded by software bootloader stub + (ie Espressif bootloader or rboot) + """ + + ROM_LOADER = ESP8266ROM + + def __init__(self, load_file=None): + super(OTAFirmwareImage, self).__init__() + self.version = 2 + if load_file is not None: + segments = self.load_common_header(load_file, ESPBOOTLOADER.IMAGE_V2_MAGIC) + if segments != ESPBOOTLOADER.IMAGE_V2_SEGMENT: + # segment count is not really segment count here, but we expect to see '4' + print('Warning: V2 header has unexpected "segment" count %d (usually 4)' % segments) + + # irom segment comes before the second header + # + # the file is saved in the image with a zero load address + # in the header, so we need to calculate a load address + irom_segment = self.load_segment(load_file, True) + # for actual mapped addr, add ESP8266ROM.IROM_MAP_START + flashing_Addr + 8 + irom_segment.addr = 0 + irom_segment.include_in_checksum = False + + first_flash_mode = self.flash_mode + first_flash_size_freq = self.flash_size_freq + first_entrypoint = self.entrypoint + # load the second header + + segments = self.load_common_header(load_file, ESPLoader.ESP_IMAGE_MAGIC) + + if first_flash_mode != self.flash_mode: + print('WARNING: Flash mode value in first header (0x%02x) disagrees with second (0x%02x). Using second value.' + % (first_flash_mode, self.flash_mode)) + if first_flash_size_freq != self.flash_size_freq: + print('WARNING: Flash size/freq value in first header (0x%02x) disagrees with second (0x%02x). Using second value.' + % (first_flash_size_freq, self.flash_size_freq)) + if first_entrypoint != self.entrypoint: + print('WARNING: Entrypoint address in first header (0x%08x) disagrees with second header (0x%08x). Using second value.' + % (first_entrypoint, self.entrypoint)) + + # load all the usual segments + for _ in range(segments): + self.load_segment(load_file) + self.checksum = self.read_checksum(load_file) + + def default_output_name(self, input_file): + """ Derive a default output name from the ELF name. """ + irom_segment = self.get_irom_segment() + if irom_segment is not None: + irom_offs = irom_segment.addr - ESP8266ROM.IROM_MAP_START + else: + irom_offs = 0 + return "%s-0x%05x.bin" % (os.path.splitext(input_file)[0], + irom_offs & ~(ESPLoader.FLASH_SECTOR_SIZE - 1)) + + def save(self, filename): + with open(filename, 'wb') as f: + # Save first header for irom0 segment + f.write(struct.pack(b'<BBBBI', ESPBOOTLOADER.IMAGE_V2_MAGIC, ESPBOOTLOADER.IMAGE_V2_SEGMENT, + self.flash_mode, self.flash_size_freq, self.entrypoint)) + + irom_segment = self.get_irom_segment() + if irom_segment is not None: + # save irom0 segment, make sure it has load addr 0 in the file + irom_segment = irom_segment.copy_with_new_addr(0) + self.save_segment(f, irom_segment) + + # second header, matches V1 header and contains loadable segments + normal_segments = self.get_non_irom_segments() + self.write_common_header(f, normal_segments) + checksum = ESPLoader.ESP_CHECKSUM_MAGIC + for segment in normal_segments: + checksum = self.save_segment(f, segment, checksum) + self.append_checksum(f, checksum) + + +class ESP32FirmwareImage(BaseFirmwareImage): + """ ESP32 firmware image is very similar to V1 ESP8266 image, + except with an additional 16 byte reserved header at top of image, + and because of new flash mapping capabilities the flash-mapped regions + can be placed in the normal image (just @ 64kB padded offsets). + """ + + ROM_LOADER = ESP32ROM + + # ROM bootloader will read the wp_pin field if SPI flash + # pins are remapped via flash. IDF actually enables QIO only + # from software bootloader, so this can be ignored. But needs + # to be set to this value so ROM bootloader will skip it. + WP_PIN_DISABLED = 0xEE + + EXTENDED_HEADER_STRUCT_FMT = "B" * 16 + + def __init__(self, load_file=None): + super(ESP32FirmwareImage, self).__init__() + self.flash_mode = 0 + self.flash_size_freq = 0 + self.version = 1 + self.wp_pin = self.WP_PIN_DISABLED + # SPI pin drive levels + self.clk_drv = 0 + self.q_drv = 0 + self.d_drv = 0 + self.cs_drv = 0 + self.hd_drv = 0 + self.wp_drv = 0 + + self.append_digest = True + + if load_file is not None: + start = load_file.tell() + + segments = self.load_common_header(load_file, ESPLoader.ESP_IMAGE_MAGIC) + self.load_extended_header(load_file) + + for _ in range(segments): + self.load_segment(load_file) + self.checksum = self.read_checksum(load_file) + + if self.append_digest: + end = load_file.tell() + self.stored_digest = load_file.read(32) + load_file.seek(start) + calc_digest = hashlib.sha256() + calc_digest.update(load_file.read(end - start)) + self.calc_digest = calc_digest.digest() # TODO: decide what to do here? + + def is_flash_addr(self, addr): + return (ESP32ROM.IROM_MAP_START <= addr < ESP32ROM.IROM_MAP_END) \ + or (ESP32ROM.DROM_MAP_START <= addr < ESP32ROM.DROM_MAP_END) + + def default_output_name(self, input_file): + """ Derive a default output name from the ELF name. """ + return "%s.bin" % (os.path.splitext(input_file)[0]) + + def warn_if_unusual_segment(self, offset, size, is_irom_segment): + pass # TODO: add warnings for ESP32 segment offset/size combinations that are wrong + + def save(self, filename): + total_segments = 0 + with io.BytesIO() as f: # write file to memory first + self.write_common_header(f, self.segments) + + # first 4 bytes of header are read by ROM bootloader for SPI + # config, but currently unused + self.save_extended_header(f) + + checksum = ESPLoader.ESP_CHECKSUM_MAGIC + + # split segments into flash-mapped vs ram-loaded, and take copies so we can mutate them + flash_segments = [copy.deepcopy(s) for s in sorted(self.segments, key=lambda s:s.addr) if self.is_flash_addr(s.addr)] + ram_segments = [copy.deepcopy(s) for s in sorted(self.segments, key=lambda s:s.addr) if not self.is_flash_addr(s.addr)] + + IROM_ALIGN = 65536 + + # check for multiple ELF sections that are mapped in the same flash mapping region. + # this is usually a sign of a broken linker script, but if you have a legitimate + # use case then let us know (we can merge segments here, but as a rule you probably + # want to merge them in your linker script.) + if len(flash_segments) > 0: + last_addr = flash_segments[0].addr + for segment in flash_segments[1:]: + if segment.addr // IROM_ALIGN == last_addr // IROM_ALIGN: + raise FatalError(("Segment loaded at 0x%08x lands in same 64KB flash mapping as segment loaded at 0x%08x. " + + "Can't generate binary. Suggest changing linker script or ELF to merge sections.") % + (segment.addr, last_addr)) + last_addr = segment.addr + + def get_alignment_data_needed(segment): + # Actual alignment (in data bytes) required for a segment header: positioned so that + # after we write the next 8 byte header, file_offs % IROM_ALIGN == segment.addr % IROM_ALIGN + # + # (this is because the segment's vaddr may not be IROM_ALIGNed, more likely is aligned + # IROM_ALIGN+0x18 to account for the binary file header + align_past = (segment.addr % IROM_ALIGN) - self.SEG_HEADER_LEN + pad_len = (IROM_ALIGN - (f.tell() % IROM_ALIGN)) + align_past + if pad_len == 0 or pad_len == IROM_ALIGN: + return 0 # already aligned + + # subtract SEG_HEADER_LEN a second time, as the padding block has a header as well + pad_len -= self.SEG_HEADER_LEN + if pad_len < 0: + pad_len += IROM_ALIGN + return pad_len + + # try to fit each flash segment on a 64kB aligned boundary + # by padding with parts of the non-flash segments... + while len(flash_segments) > 0: + segment = flash_segments[0] + pad_len = get_alignment_data_needed(segment) + if pad_len > 0: # need to pad + if len(ram_segments) > 0 and pad_len > self.SEG_HEADER_LEN: + pad_segment = ram_segments[0].split_image(pad_len) + if len(ram_segments[0].data) == 0: + ram_segments.pop(0) + else: + pad_segment = ImageSegment(0, b'\x00' * pad_len, f.tell()) + checksum = self.save_segment(f, pad_segment, checksum) + total_segments += 1 + else: + # write the flash segment + assert (f.tell() + 8) % IROM_ALIGN == segment.addr % IROM_ALIGN + checksum = self.save_segment(f, segment, checksum) + flash_segments.pop(0) + total_segments += 1 + + # flash segments all written, so write any remaining RAM segments + for segment in ram_segments: + checksum = self.save_segment(f, segment, checksum) + total_segments += 1 + + # done writing segments + self.append_checksum(f, checksum) + # kinda hacky: go back to the initial header and write the new segment count + # that includes padding segments. This header is not checksummed + image_length = f.tell() + f.seek(1) + try: + f.write(chr(total_segments)) + except TypeError: # Python 3 + f.write(bytes([total_segments])) + + if self.append_digest: + # calculate the SHA256 of the whole file and append it + f.seek(0) + digest = hashlib.sha256() + digest.update(f.read(image_length)) + f.write(digest.digest()) + + with open(filename, 'wb') as real_file: + real_file.write(f.getvalue()) + + def load_extended_header(self, load_file): + def split_byte(n): + return (n & 0x0F, (n >> 4) & 0x0F) + + fields = list(struct.unpack(self.EXTENDED_HEADER_STRUCT_FMT, load_file.read(16))) + + self.wp_pin = fields[0] + + # SPI pin drive stengths are two per byte + self.clk_drv, self.q_drv = split_byte(fields[1]) + self.d_drv, self.cs_drv = split_byte(fields[2]) + self.hd_drv, self.wp_drv = split_byte(fields[3]) + + if fields[15] in [0, 1]: + self.append_digest = (fields[15] == 1) + else: + raise RuntimeError("Invalid value for append_digest field (0x%02x). Should be 0 or 1.", fields[15]) + + # remaining fields in the middle should all be zero + if any(f for f in fields[4:15] if f != 0): + print("Warning: some reserved header fields have non-zero values. This image may be from a newer esptool.py?") + + def save_extended_header(self, save_file): + def join_byte(ln,hn): + return (ln & 0x0F) + ((hn & 0x0F) << 4) + + append_digest = 1 if self.append_digest else 0 + + fields = [self.wp_pin, + join_byte(self.clk_drv, self.q_drv), + join_byte(self.d_drv, self.cs_drv), + join_byte(self.hd_drv, self.wp_drv)] + fields += [0] * 11 + fields += [append_digest] + + packed = struct.pack(self.EXTENDED_HEADER_STRUCT_FMT, *fields) + save_file.write(packed) + + +class ELFFile(object): + SEC_TYPE_PROGBITS = 0x01 + SEC_TYPE_STRTAB = 0x03 + + LEN_SEC_HEADER = 0x28 + + def __init__(self, name): + # Load sections from the ELF file + self.name = name + with open(self.name, 'rb') as f: + self._read_elf_file(f) + + def get_section(self, section_name): + for s in self.sections: + if s.name == section_name: + return s + raise ValueError("No section %s in ELF file" % section_name) + + def _read_elf_file(self, f): + # read the ELF file header + LEN_FILE_HEADER = 0x34 + try: + (ident,_type,machine,_version, + self.entrypoint,_phoff,shoff,_flags, + _ehsize, _phentsize,_phnum, shentsize, + shnum,shstrndx) = struct.unpack("<16sHHLLLLLHHHHHH", f.read(LEN_FILE_HEADER)) + except struct.error as e: + raise FatalError("Failed to read a valid ELF header from %s: %s" % (self.name, e)) + + if byte(ident, 0) != 0x7f or ident[1:4] != b'ELF': + raise FatalError("%s has invalid ELF magic header" % self.name) + if machine != 0x5e: + raise FatalError("%s does not appear to be an Xtensa ELF file. e_machine=%04x" % (self.name, machine)) + if shentsize != self.LEN_SEC_HEADER: + raise FatalError("%s has unexpected section header entry size 0x%x (not 0x28)" % (self.name, shentsize, self.LEN_SEC_HEADER)) + if shnum == 0: + raise FatalError("%s has 0 section headers" % (self.name)) + self._read_sections(f, shoff, shnum, shstrndx) + + def _read_sections(self, f, section_header_offs, section_header_count, shstrndx): + f.seek(section_header_offs) + len_bytes = section_header_count * self.LEN_SEC_HEADER + section_header = f.read(len_bytes) + if len(section_header) == 0: + raise FatalError("No section header found at offset %04x in ELF file." % section_header_offs) + if len(section_header) != (len_bytes): + raise FatalError("Only read 0x%x bytes from section header (expected 0x%x.) Truncated ELF file?" % (len(section_header), len_bytes)) + + # walk through the section header and extract all sections + section_header_offsets = range(0, len(section_header), self.LEN_SEC_HEADER) + + def read_section_header(offs): + name_offs,sec_type,_flags,lma,sec_offs,size = struct.unpack_from("<LLLLLL", section_header[offs:]) + return (name_offs, sec_type, lma, size, sec_offs) + all_sections = [read_section_header(offs) for offs in section_header_offsets] + prog_sections = [s for s in all_sections if s[1] == ELFFile.SEC_TYPE_PROGBITS] + + # search for the string table section + if not (shstrndx * self.LEN_SEC_HEADER) in section_header_offsets: + raise FatalError("ELF file has no STRTAB section at shstrndx %d" % shstrndx) + _,sec_type,_,sec_size,sec_offs = read_section_header(shstrndx * self.LEN_SEC_HEADER) + if sec_type != ELFFile.SEC_TYPE_STRTAB: + print('WARNING: ELF file has incorrect STRTAB section type 0x%02x' % sec_type) + f.seek(sec_offs) + string_table = f.read(sec_size) + + # build the real list of ELFSections by reading the actual section names from the + # string table section, and actual data for each section from the ELF file itself + def lookup_string(offs): + raw = string_table[offs:] + return raw[:raw.index(b'\x00')] + + def read_data(offs,size): + f.seek(offs) + return f.read(size) + + prog_sections = [ELFSection(lookup_string(n_offs), lma, read_data(offs, size)) for (n_offs, _type, lma, size, offs) in prog_sections + if lma != 0] + self.sections = prog_sections + + +def slip_reader(port): + """Generator to read SLIP packets from a serial port. + Yields one full SLIP packet at a time, raises exception on timeout or invalid data. + + Designed to avoid too many calls to serial.read(1), which can bog + down on slow systems. + """ + partial_packet = None + in_escape = False + while True: + waiting = port.inWaiting() + read_bytes = port.read(1 if waiting == 0 else waiting) + if read_bytes == b'': + raise FatalError("Timed out waiting for packet %s" % ("header" if partial_packet is None else "content")) + for b in read_bytes: + + if type(b) is int: + b = bytes([b]) # python 2/3 compat + + if partial_packet is None: # waiting for packet header + if b == b'\xc0': + partial_packet = b"" + else: + raise FatalError('Invalid head of packet (%r)' % b) + elif in_escape: # part-way through escape sequence + in_escape = False + if b == b'\xdc': + partial_packet += b'\xc0' + elif b == b'\xdd': + partial_packet += b'\xdb' + else: + raise FatalError('Invalid SLIP escape (%r%r)' % (b'\xdb', b)) + elif b == b'\xdb': # start of escape sequence + in_escape = True + elif b == b'\xc0': # end of packet + yield partial_packet + partial_packet = None + else: # normal byte in packet + partial_packet += b + + +def arg_auto_int(x): + return int(x, 0) + + +def div_roundup(a, b): + """ Return a/b rounded up to nearest integer, + equivalent result to int(math.ceil(float(int(a)) / float(int(b))), only + without possible floating point accuracy errors. + """ + return (int(a) + int(b) - 1) // int(b) + + +def align_file_position(f, size): + """ Align the position in the file to the next block of specified size """ + align = (size - 1) - (f.tell() % size) + f.seek(align, 1) + + +def flash_size_bytes(size): + """ Given a flash size of the type passed in args.flash_size + (ie 512KB or 1MB) then return the size in bytes. + """ + if "MB" in size: + return int(size[:size.index("MB")]) * 1024 * 1024 + elif "KB" in size: + return int(size[:size.index("KB")]) * 1024 + else: + raise FatalError("Unknown size %s" % size) + + +def hexify(s): + if not PYTHON2: + return ''.join('%02X' % c for c in s) + else: + return ''.join('%02X' % ord(c) for c in s) + + +def unhexify(hs): + s = bytes() + + for i in range(0, len(hs) - 1, 2): + hex_string = hs[i:i + 2] + + if not PYTHON2: + s += bytes([int(hex_string, 16)]) + else: + s += chr(int(hex_string, 16)) + + return s + + +def pad_to(data, alignment, pad_character=b'\xFF'): + """ Pad to the next alignment boundary """ + pad_mod = len(data) % alignment + if pad_mod != 0: + data += pad_character * (alignment - pad_mod) + return data + + +class FatalError(RuntimeError): + """ + Wrapper class for runtime errors that aren't caused by internal bugs, but by + ESP8266 responses or input content. + """ + def __init__(self, message): + RuntimeError.__init__(self, message) + + @staticmethod + def WithResult(message, result): + """ + Return a fatal error object that appends the hex values of + 'result' as a string formatted argument. + """ + message += " (result was %s)" % hexify(result) + return FatalError(message) + + +class NotImplementedInROMError(FatalError): + """ + Wrapper class for the error thrown when a particular ESP bootloader function + is not implemented in the ROM bootloader. + """ + def __init__(self, bootloader, func): + FatalError.__init__(self, "%s ROM does not support function %s." % (bootloader.CHIP_NAME, func.__name__)) + +# "Operation" commands, executable at command line. One function each +# +# Each function takes either two args (<ESPLoader instance>, <args>) or a single <args> +# argument. + + +def load_ram(esp, args): + image = LoadFirmwareImage(esp, args.filename) + + print('RAM boot...') + for (offset, size, data) in image.segments: + print('Downloading %d bytes at %08x...' % (size, offset), end=' ') + sys.stdout.flush() + esp.mem_begin(size, div_roundup(size, esp.ESP_RAM_BLOCK), esp.ESP_RAM_BLOCK, offset) + + seq = 0 + while len(data) > 0: + esp.mem_block(data[0:esp.ESP_RAM_BLOCK], seq) + data = data[esp.ESP_RAM_BLOCK:] + seq += 1 + print('done!') + + print('All segments done, executing at %08x' % image.entrypoint) + esp.mem_finish(image.entrypoint) + + +def read_mem(esp, args): + print('0x%08x = 0x%08x' % (args.address, esp.read_reg(args.address))) + + +def write_mem(esp, args): + esp.write_reg(args.address, args.value, args.mask, 0) + print('Wrote %08x, mask %08x to %08x' % (args.value, args.mask, args.address)) + + +def dump_mem(esp, args): + f = open(args.filename, 'wb') + for i in range(args.size // 4): + d = esp.read_reg(args.address + (i * 4)) + f.write(struct.pack(b'<I', d)) + if f.tell() % 1024 == 0: + print('\r%d bytes read... (%d %%)' % (f.tell(), + f.tell() * 100 // args.size), + end=' ') + sys.stdout.flush() + print('Done!') + + +def detect_flash_size(esp, args): + if args.flash_size == 'detect': + flash_id = esp.flash_id() + size_id = flash_id >> 16 + args.flash_size = DETECTED_FLASH_SIZES.get(size_id) + if args.flash_size is None: + print('Warning: Could not auto-detect Flash size (FlashID=0x%x, SizeID=0x%x), defaulting to 4MB' % (flash_id, size_id)) + args.flash_size = '4MB' + else: + print('Auto-detected Flash size:', args.flash_size) + + +def _update_image_flash_params(esp, address, args, image): + """ Modify the flash mode & size bytes if this looks like an executable bootloader image """ + if len(image) < 8: + return image # not long enough to be a bootloader image + + # unpack the (potential) image header + magic, _, flash_mode, flash_size_freq = struct.unpack("BBBB", image[:4]) + if address != esp.BOOTLOADER_FLASH_OFFSET or magic != esp.ESP_IMAGE_MAGIC: + return image # not flashing a bootloader, so don't modify this + + if args.flash_mode != 'keep': + flash_mode = {'qio':0, 'qout':1, 'dio':2, 'dout': 3}[args.flash_mode] + + flash_freq = flash_size_freq & 0x0F + if args.flash_freq != 'keep': + flash_freq = {'40m':0, '26m':1, '20m':2, '80m': 0xf}[args.flash_freq] + + flash_size = flash_size_freq & 0xF0 + if args.flash_size != 'keep': + flash_size = esp.parse_flash_size_arg(args.flash_size) + + flash_params = struct.pack(b'BB', flash_mode, flash_size + flash_freq) + if flash_params != image[2:4]: + print('Flash params set to 0x%04x' % struct.unpack(">H", flash_params)) + image = image[0:2] + flash_params + image[4:] + return image + + +def write_flash(esp, args): + # set args.compress based on default behaviour: + # -> if either --compress or --no-compress is set, honour that + # -> otherwise, set --compress unless --no-stub is set + if args.compress is None and not args.no_compress: + args.compress = not args.no_stub + + # verify file sizes fit in flash + flash_end = flash_size_bytes(args.flash_size) + for address, argfile in args.addr_filename: + argfile.seek(0,2) # seek to end + if address + argfile.tell() > flash_end: + raise FatalError(("File %s (length %d) at offset %d will not fit in %d bytes of flash. " + + "Use --flash-size argument, or change flashing address.") + % (argfile.name, argfile.tell(), address, flash_end)) + argfile.seek(0) + + for address, argfile in args.addr_filename: + if args.no_stub: + print('Erasing flash...') + image = pad_to(argfile.read(), 4) + image = _update_image_flash_params(esp, address, args, image) + calcmd5 = hashlib.md5(image).hexdigest() + uncsize = len(image) + if args.compress: + uncimage = image + image = zlib.compress(uncimage, 9) + ratio = uncsize / len(image) + blocks = esp.flash_defl_begin(uncsize, len(image), address) + else: + ratio = 1.0 + blocks = esp.flash_begin(uncsize, address) + argfile.seek(0) # in case we need it again + seq = 0 + written = 0 + t = time.time() + esp._port.timeout = min(DEFAULT_TIMEOUT * ratio, + CHIP_ERASE_TIMEOUT * 2) + while len(image) > 0: + print('\rWriting at 0x%08x... (%d %%)' % (address + seq * esp.FLASH_WRITE_SIZE, 100 * (seq + 1) // blocks), end='') + sys.stdout.flush() + block = image[0:esp.FLASH_WRITE_SIZE] + if args.compress: + esp.flash_defl_block(block, seq) + else: + # Pad the last block + block = block + b'\xff' * (esp.FLASH_WRITE_SIZE - len(block)) + esp.flash_block(block, seq) + image = image[esp.FLASH_WRITE_SIZE:] + seq += 1 + written += len(block) + t = time.time() - t + speed_msg = "" + if args.compress: + if t > 0.0: + speed_msg = " (effective %.1f kbit/s)" % (uncsize / t * 8 / 1000) + print('\rWrote %d bytes (%d compressed) at 0x%08x in %.1f seconds%s...' % (uncsize, written, address, t, speed_msg)) + else: + if t > 0.0: + speed_msg = " (%.1f kbit/s)" % (written / t * 8 / 1000) + print('\rWrote %d bytes at 0x%08x in %.1f seconds%s...' % (written, address, t, speed_msg)) + try: + res = esp.flash_md5sum(address, uncsize) + if res != calcmd5: + print('File md5: %s' % calcmd5) + print('Flash md5: %s' % res) + print('MD5 of 0xFF is %s' % (hashlib.md5(b'\xFF' * uncsize).hexdigest())) + raise FatalError("MD5 of file does not match data in flash!") + else: + print('Hash of data verified.') + except NotImplementedInROMError: + pass + esp._port.timeout = DEFAULT_TIMEOUT + + print('\nLeaving...') + + if esp.IS_STUB: + # skip sending flash_finish to ROM loader here, + # as it causes the loader to exit and run user code + esp.flash_begin(0, 0) + if args.compress: + esp.flash_defl_finish(False) + else: + esp.flash_finish(False) + + if args.verify: + print('Verifying just-written flash...') + print('(This option is deprecated, flash contents are now always read back after flashing.)') + verify_flash(esp, args) + + +def image_info(args): + image = LoadFirmwareImage(args.chip, args.filename) + print('Image version: %d' % image.version) + print('Entry point: %08x' % image.entrypoint if image.entrypoint != 0 else 'Entry point not set') + print('%d segments' % len(image.segments)) + print + idx = 0 + for seg in image.segments: + idx += 1 + print('Segment %d: %r' % (idx, seg)) + calc_checksum = image.calculate_checksum() + print('Checksum: %02x (%s)' % (image.checksum, + 'valid' if image.checksum == calc_checksum else 'invalid - calculated %02x' % calc_checksum)) + try: + digest_msg = 'Not appended' + if image.append_digest: + is_valid = image.stored_digest == image.calc_digest + digest_msg = "%s (%s)" % (hexify(image.calc_digest).lower(), + "valid" if is_valid else "invalid") + print('Validation Hash: %s' % digest_msg) + except AttributeError: + pass # ESP8266 image has no append_digest field + + +def make_image(args): + image = ESPFirmwareImage() + if len(args.segfile) == 0: + raise FatalError('No segments specified') + if len(args.segfile) != len(args.segaddr): + raise FatalError('Number of specified files does not match number of specified addresses') + for (seg, addr) in zip(args.segfile, args.segaddr): + data = open(seg, 'rb').read() + image.segments.append(ImageSegment(addr, data)) + image.entrypoint = args.entrypoint + image.save(args.output) + + +def elf2image(args): + e = ELFFile(args.input) + if args.chip == 'auto': # Default to ESP8266 for backwards compatibility + print("Creating image for ESP8266...") + args.chip == 'esp8266' + + if args.chip == 'esp32': + image = ESP32FirmwareImage() + elif args.version == '1': # ESP8266 + image = ESPFirmwareImage() + else: + image = OTAFirmwareImage() + image.entrypoint = e.entrypoint + image.segments = e.sections # ELFSection is a subclass of ImageSegment + image.flash_mode = {'qio':0, 'qout':1, 'dio':2, 'dout': 3}[args.flash_mode] + image.flash_size_freq = image.ROM_LOADER.FLASH_SIZES[args.flash_size] + image.flash_size_freq += {'40m':0, '26m':1, '20m':2, '80m': 0xf}[args.flash_freq] + + if args.output is None: + args.output = image.default_output_name(args.input) + image.save(args.output) + + +def read_mac(esp, args): + mac = esp.read_mac() + + def print_mac(label, mac): + print('%s: %s' % (label, ':'.join(map(lambda x: '%02x' % x, mac)))) + print_mac("MAC", mac) + + +def chip_id(esp, args): + chipid = esp.chip_id() + print('Chip ID: 0x%08x' % chipid) + + +def erase_flash(esp, args): + print('Erasing flash (this may take a while)...') + t = time.time() + esp.erase_flash() + print('Chip erase completed successfully in %.1fs' % (time.time() - t)) + + +def erase_region(esp, args): + print('Erasing region (may be slow depending on size)...') + t = time.time() + esp.erase_region(args.address, args.size) + print('Erase completed successfully in %.1f seconds.' % (time.time() - t)) + + +def run(esp, args): + esp.run() + + +def flash_id(esp, args): + flash_id = esp.flash_id() + print('Manufacturer: %02x' % (flash_id & 0xff)) + flid_lowbyte = (flash_id >> 16) & 0xFF + print('Device: %02x%02x' % ((flash_id >> 8) & 0xff, flid_lowbyte)) + print('Detected flash size: %s' % (DETECTED_FLASH_SIZES.get(flid_lowbyte, "Unknown"))) + + +def read_flash(esp, args): + if args.no_progress: + flash_progress = None + else: + def flash_progress(progress, length): + msg = '%d (%d %%)' % (progress, progress * 100.0 / length) + padding = '\b' * len(msg) + if progress == length: + padding = '\n' + sys.stdout.write(msg + padding) + sys.stdout.flush() + t = time.time() + data = esp.read_flash(args.address, args.size, flash_progress) + t = time.time() - t + print('\rRead %d bytes at 0x%x in %.1f seconds (%.1f kbit/s)...' + % (len(data), args.address, t, len(data) / t * 8 / 1000)) + open(args.filename, 'wb').write(data) + + +def verify_flash(esp, args): + differences = False + + for address, argfile in args.addr_filename: + image = pad_to(argfile.read(), 4) + argfile.seek(0) # rewind in case we need it again + + image = _update_image_flash_params(esp, address, args, image) + + image_size = len(image) + print('Verifying 0x%x (%d) bytes @ 0x%08x in flash against %s...' % (image_size, image_size, address, argfile.name)) + # Try digest first, only read if there are differences. + digest = esp.flash_md5sum(address, image_size) + expected_digest = hashlib.md5(image).hexdigest() + if digest == expected_digest: + print('-- verify OK (digest matched)') + continue + else: + differences = True + if getattr(args, 'diff', 'no') != 'yes': + print('-- verify FAILED (digest mismatch)') + continue + + flash = esp.read_flash(address, image_size) + assert flash != image + diff = [i for i in range(image_size) if flash[i] != image[i]] + print('-- verify FAILED: %d differences, first @ 0x%08x' % (len(diff), address + diff[0])) + for d in diff: + flash_byte = flash[d] + image_byte = image[d] + if PYTHON2: + flash_byte = ord(flash_byte) + image_byte = ord(image_byte) + print(' %08x %02x %02x' % (address + d, flash_byte, image_byte)) + if differences: + raise FatalError("Verify failed.") + + +def read_flash_status(esp, args): + print('Status value: 0x%04x' % esp.read_status(args.bytes)) + + +def write_flash_status(esp, args): + fmt = "0x%%0%dx" % (args.bytes * 2) + args.value = args.value & ((1 << (args.bytes * 8)) - 1) + print(('Initial flash status: ' + fmt) % esp.read_status(args.bytes)) + print(('Setting flash status: ' + fmt) % args.value) + esp.write_status(args.value, args.bytes, args.non_volatile) + print(('After flash status: ' + fmt) % esp.read_status(args.bytes)) + + +def version(args): + print(__version__) + +# +# End of operations functions +# + + +def main(): + parser = argparse.ArgumentParser(description='esptool.py v%s - ESP8266 ROM Bootloader Utility' % __version__, prog='esptool') + + parser.add_argument('--chip', '-c', + help='Target chip type', + choices=['auto', 'esp8266', 'esp32'], + default=os.environ.get('ESPTOOL_CHIP', 'auto')) + + parser.add_argument( + '--port', '-p', + help='Serial port device', + default=os.environ.get('ESPTOOL_PORT', ESPLoader.DEFAULT_PORT)) + + parser.add_argument( + '--baud', '-b', + help='Serial port baud rate used when flashing/reading', + type=arg_auto_int, + default=os.environ.get('ESPTOOL_BAUD', ESPLoader.ESP_ROM_BAUD)) + + parser.add_argument( + '--before', + help='What to do before connecting to the chip', + choices=['default_reset', 'no_reset'], + default=os.environ.get('ESPTOOL_BEFORE', 'default_reset')) + + parser.add_argument( + '--after', '-a', + help='What to do after esptool.py is finished', + choices=['hard_reset', 'soft_reset', 'no_reset'], + default=os.environ.get('ESPTOOL_AFTER', 'hard_reset')) + + subparsers = parser.add_subparsers( + dest='operation', + help='Run esptool {command} -h for additional help') + + def add_spi_connection_arg(parent): + parent.add_argument('--spi-connection', '-sc', help='ESP32-only argument. Override default SPI Flash connection. ' + + 'Value can be SPI, HSPI or a comma-separated list of 5 I/O numbers to use for SPI flash (CLK,Q,D,HD,CS).', + action=SpiConnectionAction) + + parser_load_ram = subparsers.add_parser( + 'load_ram', + help='Download an image to RAM and execute') + parser_load_ram.add_argument('filename', help='Firmware image') + + parser_dump_mem = subparsers.add_parser( + 'dump_mem', + help='Dump arbitrary memory to disk') + parser_dump_mem.add_argument('address', help='Base address', type=arg_auto_int) + parser_dump_mem.add_argument('size', help='Size of region to dump', type=arg_auto_int) + parser_dump_mem.add_argument('filename', help='Name of binary dump') + + parser_read_mem = subparsers.add_parser( + 'read_mem', + help='Read arbitrary memory location') + parser_read_mem.add_argument('address', help='Address to read', type=arg_auto_int) + + parser_write_mem = subparsers.add_parser( + 'write_mem', + help='Read-modify-write to arbitrary memory location') + parser_write_mem.add_argument('address', help='Address to write', type=arg_auto_int) + parser_write_mem.add_argument('value', help='Value', type=arg_auto_int) + parser_write_mem.add_argument('mask', help='Mask of bits to write', type=arg_auto_int) + + def add_spi_flash_subparsers(parent, is_elf2image): + """ Add common parser arguments for SPI flash properties """ + extra_keep_args = [] if is_elf2image else ['keep'] + auto_detect = not is_elf2image + + parent.add_argument('--flash_freq', '-ff', help='SPI Flash frequency', + choices=extra_keep_args + ['40m', '26m', '20m', '80m'], + default=os.environ.get('ESPTOOL_FF', '40m' if is_elf2image else 'keep')) + parent.add_argument('--flash_mode', '-fm', help='SPI Flash mode', + choices=extra_keep_args + ['qio', 'qout', 'dio', 'dout'], + default=os.environ.get('ESPTOOL_FM', 'qio' if is_elf2image else 'keep')) + parent.add_argument('--flash_size', '-fs', help='SPI Flash size in MegaBytes (1MB, 2MB, 4MB, 8MB, 16M)' + ' plus ESP8266-only (256KB, 512KB, 2MB-c1, 4MB-c1)', + action=FlashSizeAction, auto_detect=auto_detect, + default=os.environ.get('ESPTOOL_FS', 'detect' if auto_detect else '1MB')) + add_spi_connection_arg(parent) + + parser_write_flash = subparsers.add_parser( + 'write_flash', + help='Write a binary blob to flash') + parser_write_flash.add_argument('addr_filename', metavar='<address> <filename>', help='Address followed by binary filename, separated by space', + action=AddrFilenamePairAction) + add_spi_flash_subparsers(parser_write_flash, is_elf2image=False) + parser_write_flash.add_argument('--no-progress', '-p', help='Suppress progress output', action="store_true") + parser_write_flash.add_argument('--verify', help='Verify just-written data on flash ' + + '(mostly superfluous, data is read back during flashing)', action='store_true') + compress_args = parser_write_flash.add_mutually_exclusive_group(required=False) + compress_args.add_argument('--compress', '-z', help='Compress data in transfer (default unless --no-stub is specified)',action="store_true", default=None) + compress_args.add_argument('--no-compress', '-u', help='Disable data compression during transfer (default if --no-stub is specified)',action="store_true") + + subparsers.add_parser( + 'run', + help='Run application code in flash') + + parser_image_info = subparsers.add_parser( + 'image_info', + help='Dump headers from an application image') + parser_image_info.add_argument('filename', help='Image file to parse') + + parser_make_image = subparsers.add_parser( + 'make_image', + help='Create an application image from binary files') + parser_make_image.add_argument('output', help='Output image file') + parser_make_image.add_argument('--segfile', '-f', action='append', help='Segment input file') + parser_make_image.add_argument('--segaddr', '-a', action='append', help='Segment base address', type=arg_auto_int) + parser_make_image.add_argument('--entrypoint', '-e', help='Address of entry point', type=arg_auto_int, default=0) + + parser_elf2image = subparsers.add_parser( + 'elf2image', + help='Create an application image from ELF file') + parser_elf2image.add_argument('input', help='Input ELF file') + parser_elf2image.add_argument('--output', '-o', help='Output filename prefix (for version 1 image), or filename (for version 2 single image)', type=str) + parser_elf2image.add_argument('--version', '-e', help='Output image version', choices=['1','2'], default='1') + + add_spi_flash_subparsers(parser_elf2image, is_elf2image=True) + + subparsers.add_parser( + 'read_mac', + help='Read MAC address from OTP ROM') + + subparsers.add_parser( + 'chip_id', + help='Read Chip ID from OTP ROM') + + parser_flash_id = subparsers.add_parser( + 'flash_id', + help='Read SPI flash manufacturer and device ID') + add_spi_connection_arg(parser_flash_id) + + parser_read_status = subparsers.add_parser( + 'read_flash_status', + help='Read SPI flash status register') + + add_spi_connection_arg(parser_read_status) + parser_read_status.add_argument('--bytes', help='Number of bytes to read (1-3)', type=int, choices=[1,2,3], default=2) + + parser_write_status = subparsers.add_parser( + 'write_flash_status', + help='Write SPI flash status register') + + add_spi_connection_arg(parser_write_status) + parser_write_status.add_argument('--non-volatile', help='Write non-volatile bits (use with caution)', action='store_true') + parser_write_status.add_argument('--bytes', help='Number of status bytes to write (1-3)', type=int, choices=[1,2,3], default=2) + parser_write_status.add_argument('value', help='New value', type=arg_auto_int) + + parser_read_flash = subparsers.add_parser( + 'read_flash', + help='Read SPI flash content') + add_spi_connection_arg(parser_read_flash) + parser_read_flash.add_argument('address', help='Start address', type=arg_auto_int) + parser_read_flash.add_argument('size', help='Size of region to dump', type=arg_auto_int) + parser_read_flash.add_argument('filename', help='Name of binary dump') + parser_read_flash.add_argument('--no-progress', '-p', help='Suppress progress output', action="store_true") + + parser_verify_flash = subparsers.add_parser( + 'verify_flash', + help='Verify a binary blob against flash') + parser_verify_flash.add_argument('addr_filename', help='Address and binary file to verify there, separated by space', + action=AddrFilenamePairAction) + parser_verify_flash.add_argument('--diff', '-d', help='Show differences', + choices=['no', 'yes'], default='no') + add_spi_flash_subparsers(parser_verify_flash, is_elf2image=False) + + parser_erase_flash = subparsers.add_parser( + 'erase_flash', + help='Perform Chip Erase on SPI flash') + add_spi_connection_arg(parser_erase_flash) + + parser_erase_region = subparsers.add_parser( + 'erase_region', + help='Erase a region of the flash') + add_spi_connection_arg(parser_erase_region) + parser_erase_region.add_argument('address', help='Start address (must be multiple of 4096)', type=arg_auto_int) + parser_erase_region.add_argument('size', help='Size of region to erase (must be multiple of 4096)', type=arg_auto_int) + + subparsers.add_parser( + 'version', help='Print esptool version') + + # internal sanity check - every operation matches a module function of the same name + for operation in subparsers.choices.keys(): + assert operation in globals(), "%s should be a module function" % operation + + expand_file_arguments() + + args = parser.parse_args() + + print('esptool.py v%s' % __version__) + + args.no_stub = True + + # operation function can take 1 arg (args), 2 args (esp, arg) + # or be a member function of the ESPLoader class. + + if args.operation is None: + parser.print_help() + sys.exit(1) + + operation_func = globals()[args.operation] + operation_args,_,_,_ = inspect.getargspec(operation_func) + if operation_args[0] == 'esp': # operation function takes an ESPLoader connection object + initial_baud = min(ESPLoader.ESP_ROM_BAUD, args.baud) # don't sync faster than the default baud rate + if args.chip == 'auto': + esp = ESPLoader.detect_chip(args.port, initial_baud, args.before) + else: + chip_class = { + 'esp8266': ESP8266ROM, + 'esp32': ESP32ROM, + }[args.chip] + esp = chip_class(args.port, initial_baud) + esp.connect(args.before) + + print("Chip is %s" % (esp.get_chip_description())) + + if not args.no_stub: + esp = esp.run_stub() + + if args.baud > initial_baud: + try: + esp.change_baud(args.baud) + except NotImplementedInROMError: + print("WARNING: ROM doesn't support changing baud rate. Keeping initial baud rate %d" % initial_baud) + + # override common SPI flash parameter stuff if configured to do so + if hasattr(args, "spi_connection") and args.spi_connection is not None: + if esp.CHIP_NAME != "ESP32": + raise FatalError("Chip %s does not support --spi-connection option." % esp.CHIP_NAME) + print("Configuring SPI flash mode...") + esp.flash_spi_attach(args.spi_connection) + elif args.no_stub: + print("Enabling default SPI flash mode...") + # ROM loader doesn't enable flash unless we explicitly do it + esp.flash_spi_attach(0) + + if hasattr(args, "flash_size"): + print("Configuring flash size...") + detect_flash_size(esp, args) + esp.flash_set_parameters(flash_size_bytes(args.flash_size)) + + operation_func(esp, args) + + # finish execution based on args.after + if args.after == 'hard_reset': + print('Hard resetting...') + esp.hard_reset() + elif args.after == 'soft_reset': + print('Soft resetting...') + # flash_finish will trigger a soft reset + esp.soft_reset(False) + else: + print('Staying in bootloader.') + if esp.IS_STUB: + esp.soft_reset(True) # exit stub back to ROM loader + + else: + operation_func(args) + + +def expand_file_arguments(): + """ Any argument starting with "@" gets replaced with all values read from a text file. + Text file arguments can be split by newline or by space. + Values are added "as-is", as if they were specified in this order on the command line. + """ + new_args = [] + expanded = False + for arg in sys.argv: + if arg.startswith("@"): + expanded = True + with open(arg[1:],"r") as f: + for line in f.readlines(): + new_args += shlex.split(line) + else: + new_args.append(arg) + if expanded: + print("esptool.py %s" % (" ".join(new_args[1:]))) + sys.argv = new_args + + +class FlashSizeAction(argparse.Action): + """ Custom flash size parser class to support backwards compatibility with megabit size arguments. + + (At next major relase, remove deprecated sizes and this can become a 'normal' choices= argument again.) + """ + def __init__(self, option_strings, dest, nargs=1, auto_detect=False, **kwargs): + super(FlashSizeAction, self).__init__(option_strings, dest, nargs, **kwargs) + self._auto_detect = auto_detect + + def __call__(self, parser, namespace, values, option_string=None): + try: + value = { + '2m': '256KB', + '4m': '512KB', + '8m': '1MB', + '16m': '2MB', + '32m': '4MB', + '16m-c1': '2MB-c1', + '32m-c1': '4MB-c1', + }[values[0]] + print("WARNING: Flash size arguments in megabits like '%s' are deprecated." % (values[0])) + print("Please use the equivalent size '%s'." % (value)) + print("Megabit arguments may be removed in a future release.") + except KeyError: + value = values[0] + + known_sizes = dict(ESP8266ROM.FLASH_SIZES) + known_sizes.update(ESP32ROM.FLASH_SIZES) + if self._auto_detect: + known_sizes['detect'] = 'detect' + if value not in known_sizes: + raise argparse.ArgumentError(self, '%s is not a known flash size. Known sizes: %s' % (value, ", ".join(known_sizes.keys()))) + setattr(namespace, self.dest, value) + + +class SpiConnectionAction(argparse.Action): + """ Custom action to parse 'spi connection' override. Values are SPI, HSPI, or a sequence of 5 pin numbers separated by commas. + """ + def __call__(self, parser, namespace, value, option_string=None): + if value.upper() == "SPI": + value = 0 + elif value.upper() == "HSPI": + value = 1 + elif "," in value: + values = value.split(",") + if len(values) != 5: + raise argparse.ArgumentError(self, '%s is not a valid list of comma-separate pin numbers. Must be 5 numbers - CLK,Q,D,HD,CS.' % value) + try: + values = tuple(int(v,0) for v in values) + except ValueError: + raise argparse.ArgumentError(self, '%s is not a valid argument. All pins must be numeric values' % values) + if any([v for v in values if v > 33 or v < 0]): + raise argparse.ArgumentError(self, 'Pin numbers must be in the range 0-33.') + # encode the pin numbers as a 32-bit integer with packed 6-bit values, the same way ESP32 ROM takes them + # TODO: make this less ESP32 ROM specific somehow... + clk,q,d,hd,cs = values + value = (hd << 24) | (cs << 18) | (d << 12) | (q << 6) | clk + else: + raise argparse.ArgumentError(self, '%s is not a valid spi-connection value. ' + + 'Values are SPI, HSPI, or a sequence of 5 pin numbers CLK,Q,D,HD,CS).' % values) + setattr(namespace, self.dest, value) + + +class AddrFilenamePairAction(argparse.Action): + """ Custom parser class for the address/filename pairs passed as arguments """ + def __init__(self, option_strings, dest, nargs='+', **kwargs): + super(AddrFilenamePairAction, self).__init__(option_strings, dest, nargs, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + # validate pair arguments + pairs = [] + for i in range(0,len(values),2): + try: + address = int(values[i],0) + except ValueError as e: + raise argparse.ArgumentError(self,'Address "%s" must be a number' % values[i]) + try: + argfile = open(values[i + 1], 'rb') + except IOError as e: + raise argparse.ArgumentError(self, e) + except IndexError: + raise argparse.ArgumentError(self,'Must be pairs of an address and the binary filename to write there') + pairs.append((address, argfile)) + + # Sort the addresses and check for overlapping + end = 0 + for address, argfile in sorted(pairs): + argfile.seek(0,2) # seek to end + size = argfile.tell() + argfile.seek(0) + sector_start = address & ~(ESPLoader.FLASH_SECTOR_SIZE - 1) + sector_end = ((address + size + ESPLoader.FLASH_SECTOR_SIZE - 1) & ~(ESPLoader.FLASH_SECTOR_SIZE - 1)) - 1 + if sector_start < end: + message = 'Detected overlap at address: 0x%x for file: %s' % (address, argfile.name) + raise argparse.ArgumentError(self, message) + end = sector_end + setattr(namespace, self.dest, pairs) + + +# Binary stub code purged due to DFSG + +def _main(): + try: + main() + except FatalError as e: + print('\nA fatal error occurred: %s' % e) + sys.exit(2) + + +if __name__ == '__main__': + _main() diff --git a/scripts/flash.sh b/scripts/flash.sh new file mode 100755 index 0000000..cde4473 --- /dev/null +++ b/scripts/flash.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +HERE=$(dirname "$0") + +#"$HERE"/esptool.py --port $1 write_flash --flash_size=detect 0 "$HERE"/esp8266-20171101-v1.9.3.bin +#"$HERE"/esptool.py --port $1 write_flash --flash_size=detect 0 "$HERE"/esp8266-20180511-v1.9.4.bin +set -x + + +"$HERE"/esptool.py --port $1 write_flash --flash_size=detect 0 "$HERE"/esp8266-20180511-v1.9.4.bin diff --git a/scripts/format.sh b/scripts/format.sh new file mode 100755 index 0000000..c00e4c5 --- /dev/null +++ b/scripts/format.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -x +PORT=$1 + +cat $PORT & +PRINTER_PID=$! + +# interrupt +echo -e -n "\003" >$PORT + +# Enter RAW Repl +echo -e -n "\001" >$PORT + +echo -e -n "import flashbdev\004" >$PORT +echo -e -n "import os\004" >$PORT + +sleep 0.1 + +echo -e -n "os.umount('/')\004" >$PORT +echo -e -n "os.VfsFat.mkfs(flashbdev.bdev)\004" >$PORT +echo -e -n "os.mount(flashbdev.bdev, '/')\004" >$PORT +echo -e -n "\002" >$PORT + +sleep 0.5 + +kill $PRINTER_PID diff --git a/scripts/kconfiglib.py b/scripts/kconfiglib.py new file mode 100644 index 0000000..0da2735 --- /dev/null +++ b/scripts/kconfiglib.py @@ -0,0 +1,5976 @@ +# Copyright (c) 2011-2018, Ulf Magnusson +# SPDX-License-Identifier: ISC + +""" +Overview +======== + +Kconfiglib is a Python 2/3 library for scripting and extracting information +from Kconfig (https://www.kernel.org/doc/Documentation/kbuild/kconfig-language.txt) +configuration systems. + +See the homepage at https://github.com/ulfalizer/Kconfiglib for a longer +overview. + +Using Kconfiglib on the Linux kernel with the Makefile targets +============================================================== + +For the Linux kernel, a handy interface is provided by the +scripts/kconfig/Makefile patch, which can be applied with either 'git am' or +the 'patch' utility: + + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | git am + $ wget -qO- https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch | patch -p1 + +Warning: Not passing -p1 to patch will cause the wrong file to be patched. + +Please tell me if the patch does not apply. It should be trivial to apply +manually, as it's just a block of text that needs to be inserted near the other +*conf: targets in scripts/kconfig/Makefile. + +Look further down for a motivation for the Makefile patch and for instructions +on how you can use Kconfiglib without it. + +If you do not wish to install Kconfiglib via pip, the Makefile patch is set up +so that you can also just clone Kconfiglib into the kernel root: + + $ git clone git://github.com/ulfalizer/Kconfiglib.git + $ git am Kconfiglib/makefile.patch (or 'patch -p1 < Kconfiglib/makefile.patch') + +Warning: The directory name Kconfiglib/ is significant in this case, because +it's added to PYTHONPATH by the new targets in makefile.patch. + +The targets added by the Makefile patch are described in the following +sections. + + +make [ARCH=<arch>] iscriptconfig +-------------------------------- + +This target gives an interactive Python prompt where a Kconfig instance has +been preloaded and is available in 'kconf'. To change the Python interpreter +used, pass PYTHONCMD=<executable> to make. The default is "python". + +To get a feel for the API, try evaluating and printing the symbols in +kconf.defined_syms, and explore the MenuNode menu tree starting at +kconf.top_node by following 'next' and 'list' pointers. + +The item contained in a menu node is found in MenuNode.item (note that this can +be one of the constants kconfiglib.MENU and kconfiglib.COMMENT), and all +symbols and choices have a 'nodes' attribute containing their menu nodes +(usually only one). Printing a menu node will print its item, in Kconfig +format. + +If you want to look up a symbol by name, use the kconf.syms dictionary. + + +make scriptconfig SCRIPT=<script> [SCRIPT_ARG=<arg>] +---------------------------------------------------- + +This target runs the Python script given by the SCRIPT parameter on the +configuration. sys.argv[1] holds the name of the top-level Kconfig file +(currently always "Kconfig" in practice), and sys.argv[2] holds the SCRIPT_ARG +argument, if given. + +See the examples/ subdirectory for example scripts. + + +Using Kconfiglib without the Makefile targets +============================================= + +The make targets are only needed for a trivial reason: The Kbuild makefiles +export environment variables which are referenced inside the Kconfig files and +in scripts run from the Kconfig files (via e.g. 'source +"arch/$(SRCARCH)/Kconfig" and '$(shell,...)'). + +The environment variables referenced as of writing (Linux 4.2.18-rc4) are +srctree, ARCH, SRCARCH, CC, and KERNELVERSION. + +To run Kconfiglib without the Makefile patch, you can do this: + + $ srctree=. ARCH=x86 SRCARCH=x86 CC=gcc KERNELVERSION=`make kernelversion` python(3) + >>> import kconfiglib + >>> kconf = kconfiglib.Kconfig() # filename defaults to "Kconfig" + +Search the top-level Makefile for "Additional ARCH settings" to see other +possibilities for ARCH and SRCARCH. Kconfiglib will print a warning if an unset +environment variable is referenced inside the Kconfig files. + + +Intro to symbol values +====================== + +Kconfiglib has the same assignment semantics as the C implementation. + +Any symbol can be assigned a value by the user (via Kconfig.load_config() or +Symbol.set_value()), but this user value is only respected if the symbol is +visible, which corresponds to it (currently) being visible in the menuconfig +interface. + +For symbols with prompts, the visibility of the symbol is determined by the +condition on the prompt. Symbols without prompts are never visible, so setting +a user value on them is pointless. A warning will be printed by default if +Symbol.set_value() is called on a promptless symbol. Assignments to promptless +symbols are normal within a .config file, so no similar warning will be printed +by load_config(). + +Dependencies from parents and 'if'/'depends on' are propagated to properties, +including prompts, so these two configurations are logically equivalent: + +(1) + + menu "menu" + depends on A + + if B + + config FOO + tristate "foo" if D + default y + depends on C + + endif + + endmenu + +(2) + + menu "menu" + depends on A + + config FOO + tristate "foo" if A && B && C && D + default y if A && B && C + + endmenu + +In this example, A && B && C && D (the prompt condition) needs to be non-n for +FOO to be visible (assignable). If its value is m, the symbol can only be +assigned the value m: The visibility sets an upper bound on the value that can +be assigned by the user, and any higher user value will be truncated down. + +'default' properties are independent of the visibility, though a 'default' will +often get the same condition as the prompt due to dependency propagation. +'default' properties are used if the symbol is not visible or has no user +value. + +Symbols with no user value (or that have a user value but are not visible) and +no (active) 'default' default to n for bool/tristate symbols, and to the empty +string for other symbol types. + +'select' works similarly to symbol visibility, but sets a lower bound on the +value of the symbol. The lower bound is determined by the value of the +select*ing* symbol. 'select' does not respect visibility, so non-visible +symbols can be forced to a particular (minimum) value by a select as well. + +For non-bool/tristate symbols, it only matters whether the visibility is n or +non-n: m visibility acts the same as y visibility. + +Conditions on 'default' and 'select' work in mostly intuitive ways. If the +condition is n, the 'default' or 'select' is disabled. If it is m, the +'default' or 'select' value (the value of the selecting symbol) is truncated +down to m. + +When writing a configuration with Kconfig.write_config(), only symbols that are +visible, have an (active) default, or are selected will get written out (note +that this includes all symbols that would accept user values). Kconfiglib +matches the .config format produced by the C implementations down to the +character. This eases testing. + +For a visible bool/tristate symbol FOO with value n, this line is written to +.config: + + # CONFIG_FOO is not set + +The point is to remember the user n selection (which might differ from the +default value the symbol would get), while at the same sticking to the rule +that undefined corresponds to n (.config uses Makefile format, making the line +above a comment). When the .config file is read back in, this line will be +treated the same as the following assignment: + + CONFIG_FOO=n + +In Kconfiglib, the set of (currently) assignable values for a bool/tristate +symbol appear in Symbol.assignable. For other symbol types, just check if +sym.visibility is non-0 (non-n) to see whether the user value will have an +effect. + + +Intro to the menu tree +====================== + +The menu structure, as seen in e.g. menuconfig, is represented by a tree of +MenuNode objects. The top node of the configuration corresponds to an implicit +top-level menu, the title of which is shown at the top in the standard +menuconfig interface. (The title is also available in Kconfig.mainmenu_text in +Kconfiglib.) + +The top node is found in Kconfig.top_node. From there, you can visit child menu +nodes by following the 'list' pointer, and any following menu nodes by +following the 'next' pointer. Usually, a non-None 'list' pointer indicates a +menu or Choice, but menu nodes for symbols can sometimes have a non-None 'list' +pointer too due to submenus created implicitly from dependencies. + +MenuNode.item is either a Symbol or a Choice object, or one of the constants +MENU and COMMENT. The prompt of the menu node can be found in MenuNode.prompt, +which also holds the title for menus and comments. For Symbol and Choice, +MenuNode.help holds the help text (if any, otherwise None). + +Most symbols will only have a single menu node. A symbol defined in multiple +locations will have one menu node for each location. The list of menu nodes for +a Symbol or Choice can be found in the Symbol/Choice.nodes attribute. + +Note that prompts and help texts for symbols and choices are stored in their +menu node(s) rather than in the Symbol or Choice objects themselves. This makes +it possible to define a symbol in multiple locations with a different prompt or +help text in each location. To get the help text or prompt for a symbol with a +single menu node, do sym.nodes[0].help and sym.nodes[0].prompt, respectively. +The prompt is a (text, condition) tuple, where condition determines the +visibility (see 'Intro to expressions' below). + +This organization mirrors the C implementation. MenuNode is called +'struct menu' there, but I thought "menu" was a confusing name. + +It is possible to give a Choice a name and define it in multiple locations, +hence why Choice.nodes is also a list. + +As a convenience, the properties added at a particular definition location are +available on the MenuNode itself, in e.g. MenuNode.defaults. This is helpful +when generating documentation, so that symbols/choices defined in multiple +locations can be shown with the correct properties at each location. + + +Intro to expressions +==================== + +Expressions can be evaluated with the expr_value() function and printed with +the expr_str() function (these are used internally as well). Evaluating an +expression always yields a tristate value, where n, m, and y are represented as +0, 1, and 2, respectively. + +The following table should help you figure out how expressions are represented. +A, B, C, ... are symbols (Symbol instances), NOT is the kconfiglib.NOT +constant, etc. + +Expression Representation +---------- -------------- +A A +"A" A (constant symbol) +!A (NOT, A) +A && B (AND, A, B) +A && B && C (AND, A, (AND, B, C)) +A || B (OR, A, B) +A || (B && C && D) (OR, A, (AND, B, (AND, C, D))) +A = B (EQUAL, A, B) +A != "foo" (UNEQUAL, A, foo (constant symbol)) +A && B = C && D (AND, A, (AND, (EQUAL, B, C), D)) +n Kconfig.n (constant symbol) +m Kconfig.m (constant symbol) +y Kconfig.y (constant symbol) +"y" Kconfig.y (constant symbol) + +Strings like "foo" in 'default "foo"' or 'depends on SYM = "foo"' are +represented as constant symbols, so the only values that appear in expressions +are symbols***. This mirrors the C implementation. + +***For choice symbols, the parent Choice will appear in expressions as well, +but it's usually invisible as the value interfaces of Symbol and Choice are +identical. This mirrors the C implementation and makes different choice modes +"just work". + +Manual evaluation examples: + + - The value of A && B is min(A.tri_value, B.tri_value) + + - The value of A || B is max(A.tri_value, B.tri_value) + + - The value of !A is 2 - A.tri_value + + - The value of A = B is 2 (y) if A.str_value == B.str_value, and 0 (n) + otherwise. Note that str_value is used here instead of tri_value. + + For constant (as well as undefined) symbols, str_value matches the name of + the symbol. This mirrors the C implementation and explains why + 'depends on SYM = "foo"' above works as expected. + +n/m/y are automatically converted to the corresponding constant symbols +"n"/"m"/"y" (Kconfig.n/m/y) during parsing. + +Kconfig.const_syms is a dictionary like Kconfig.syms but for constant symbols. + +If a condition is missing (e.g., <cond> when the 'if <cond>' is removed from +'default A if <cond>'), it is actually Kconfig.y. The standard __str__() +functions just avoid printing 'if y' conditions to give cleaner output. + + +Kconfig extensions +================== + +Kconfiglib implements two Kconfig extensions related to 'source': + +'source' with relative path +--------------------------- + +The library implements a custom 'rsource' statement that allows to import +Kconfig file by specifying path relative to directory of the currently parsed +file, instead of path relative to project root. +This extension is not supported by Linux kernel tools (yet). + +Consider following directory tree: + + Project + +--Kconfig + | + +--src + +--Kconfig + | + +--SubSystem1 + +--Kconfig + | + +--ModuleA + +--Kconfig + +In above example, src/SubSystem1/Kconfig imports Kconfig for ModuleA. +With default 'source' it looks like: + + source "src/SubSystem1/ModuleA/Kconfig" + +Using 'rsource' it can be rewritten as: + + rsource "ModuleA/Kconfig" + +If absolute path is given to 'rsource' then it follows behavior of 'source'. + + +Globbed sourcing +---------------- + +'source' and 'rsource' accept glob patterns, sourcing all matching Kconfig +files. They require at least one matching file, throwing a KconfigError +otherwise. + +For example, the following statement might source sub1/foofoofoo and +sub2/foobarfoo: + + source "sub[12]/foo*foo" + +The glob patterns accepted are the same as for the standard glob.glob() +function. + +Two additional statements are provided for cases where it's acceptable for a +pattern to match no files: 'osource' and 'orsource' (the o is for "optional"). + +For example, the following statements will be no-ops if neither "foo" nor any +files matching "bar*" exist: + + osource "foo" + osource "bar*" + +'orsource' does a relative optional source. + +'source' and 'osource' are analogous to 'include' and '-include' in Make. + + +Feedback +======== + +Send bug reports, suggestions, and questions to ulfalizer a.t Google's email +service, or open a ticket on the GitHub page. +""" +import errno +import glob +import os +import platform +import re +import subprocess +import sys +import textwrap + +# File layout: +# +# Public classes +# Public functions +# Internal functions +# Public global constants +# Internal global constants + +# Line length: 79 columns + +# +# Public classes +# + +class Kconfig(object): + """ + Represents a Kconfig configuration, e.g. for x86 or ARM. This is the set of + symbols, choices, and menu nodes appearing in the configuration. Creating + any number of Kconfig objects (including for different architectures) is + safe. Kconfiglib doesn't keep any global state. + + The following attributes are available. They should be treated as + read-only, and some are implemented through @property magic. + + syms: + A dictionary with all symbols in the configuration, indexed by name. Also + includes all symbols that are referenced in expressions but never + defined, except for constant (quoted) symbols. + + Undefined symbols can be recognized by Symbol.nodes being empty -- see + the 'Intro to the menu tree' section in the module docstring. + + const_syms: + A dictionary like 'syms' for constant (quoted) symbols + + named_choices: + A dictionary like 'syms' for named choices (choice FOO) + + defined_syms: + A list with all defined symbols, in the same order as they appear in the + Kconfig files. Symbols defined in multiple locations appear multiple + times. Iterating over set(defined_syms) will visit each defined symbol + once. + + choices: + A list with all choices, in the same order as they appear in the Kconfig + files. Named choices defined in multiple locations appear multiple times. + Iterating over set(choices) will visit each choice once. + + menus: + A list with all menus, in the same order as they appear in the Kconfig + files + + comments: + A list with all comments, in the same order as they appear in the Kconfig + files + + n/m/y: + The predefined constant symbols n/m/y. Also available in const_syms. + + modules: + The Symbol instance for the modules symbol. Currently hardcoded to + MODULES, which is backwards compatible. Kconfiglib will warn if + 'option modules' is set on some other symbol. Tell me if you need proper + 'option modules' support. + + 'modules' is never None. If the MODULES symbol is not explicitly defined, + its tri_value will be 0 (n), as expected. + + A simple way to enable modules is to do 'kconf.modules.set_value(2)' + (provided the MODULES symbol is defined and visible). Modules are + disabled by default in the kernel Kconfig files as of writing, though + nearly all defconfig files enable them (with 'CONFIG_MODULES=y'). + + defconfig_list: + The Symbol instance for the 'option defconfig_list' symbol, or None if no + defconfig_list symbol exists. The defconfig filename derived from this + symbol can be found in Kconfig.defconfig_filename. + + defconfig_filename: + The filename given by the defconfig_list symbol. This is taken from the + first 'default' with a satisfied condition where the specified file + exists (can be opened for reading). If a defconfig file foo/defconfig is + not found and $srctree was set when the Kconfig was created, + $srctree/foo/defconfig is looked up as well. + + 'defconfig_filename' is None if either no defconfig_list symbol exists, + or if the defconfig_list symbol has no 'default' with a satisfied + condition that specifies a file that exists. + + Gotcha: scripts/kconfig/Makefile might pass --defconfig=<defconfig> to + scripts/kconfig/conf when running e.g. 'make defconfig'. This option + overrides the defconfig_list symbol, meaning defconfig_filename might not + always match what 'make defconfig' would use. + + top_node: + The menu node (see the MenuNode class) of the implicit top-level menu. + Acts as the root of the menu tree. + + mainmenu_text: + The prompt (title) of the top menu (top_node). Defaults to "Main menu". + Can be changed with the 'mainmenu' statement (see kconfig-language.txt). + + variables: + A dictionary with all preprocessor variables, indexed by name. See the + Variable class. + + warnings: + A list of strings containing all warnings that have been generated. This + allows flexibility in how warnings are printed and processed. + + See the 'warn_to_stderr' parameter to Kconfig.__init__() and the + Kconfig.enable/disable_stderr_warnings() functions as well. Note that + warnings still get added to Kconfig.warnings when 'warn_to_stderr' is + True. + + Just as for warnings printed to stderr, only optional warnings that are + enabled will get added to Kconfig.warnings. See the various + Kconfig.enable/disable_*_warnings() functions. + + srctree: + The value of the $srctree environment variable when the configuration was + loaded, or the empty string if $srctree wasn't set. This gives nice + behavior with os.path.join(), which treats "" as the current directory, + without adding "./". + + Kconfig files are looked up relative to $srctree (unless absolute paths + are used), and .config files are looked up relative to $srctree if they + are not found in the current directory. This is used to support + out-of-tree builds. The C tools use this environment variable in the same + way. + + Changing $srctree after creating the Kconfig instance has no effect. Only + the value when the configuration is loaded matters. This avoids surprises + if multiple configurations are loaded with different values for $srctree. + + config_prefix: + The value of the $CONFIG_ environment variable when the configuration was + loaded. This is the prefix used (and expected) on symbol names in .config + files and C headers. Defaults to "CONFIG_". Used in the same way in the C + tools. + + Like for srctree, only the value of $CONFIG_ when the configuration is + loaded matters. + """ + __slots__ = ( + "_defined_syms_set", + "_encoding", + "_functions", + "_set_match", + "_unset_match", + "_warn_for_no_prompt", + "_warn_for_redun_assign", + "_warn_for_undef_assign", + "_warn_to_stderr", + "_warnings_enabled", + "choices", + "comments", + "config_prefix", + "const_syms", + "defconfig_list", + "defined_syms", + "m", + "mainmenu_text", + "menus", + "modules", + "n", + "named_choices", + "srctree", + "syms", + "top_node", + "variables", + "warnings", + "y", + + # Parsing-related + "_parsing_kconfigs", + "_file", + "_filename", + "_linenr", + "_filestack", + "_line", + "_saved_line", + "_tokens", + "_tokens_i", + "_has_tokens", + ) + + # + # Public interface + # + + def __init__(self, filename="Kconfig", warn=True, warn_to_stderr=True, + encoding="utf-8"): + """ + Creates a new Kconfig object by parsing Kconfig files. Raises + KconfigError on syntax errors. Note that Kconfig files are not the same + as .config files (which store configuration symbol values). + + If the environment variable KCONFIG_STRICT is set to "y", warnings will + be generated for all references to undefined symbols within Kconfig + files. The reason this isn't the default is that some projects (e.g. + the Linux kernel) use multiple Kconfig trees (one per architecture) + with many shared Kconfig files, leading to some safe references to + undefined symbols. + + KCONFIG_STRICT relies on literal hex values being prefixed with 0x/0X. + They are indistinguishable from references to undefined symbols + otherwise. + + KCONFIG_STRICT might enable other warnings that depend on there being + just a single Kconfig tree in the future. + + filename (default: "Kconfig"): + The Kconfig file to load. For the Linux kernel, you'll want "Kconfig" + from the top-level directory, as environment variables will make sure + the right Kconfig is included from there (arch/$SRCARCH/Kconfig as of + writing). + + If you are using Kconfiglib via 'make scriptconfig', the filename of + the base base Kconfig file will be in sys.argv[1]. It's currently + always "Kconfig" in practice. + + The $srctree environment variable is used to look up Kconfig files + referenced in Kconfig files if set. See the class documentation. + + Note: '(o)source' statements in Kconfig files always work relative to + $srctree (or the current directory if $srctree is unset), even if + 'filename' is a path with directories. This allows a subset of + Kconfig files to be loaded without breaking references to other + Kconfig files, e.g. by doing Kconfig("./sub/Kconfig"). sub/Kconfig + might expect to be sourced by ./Kconfig. + + warn (default: True): + True if warnings related to this configuration should be generated. + This can be changed later with Kconfig.enable/disable_warnings(). It + is provided as a constructor argument since warnings might be + generated during parsing. + + See the other Kconfig.enable_*_warnings() functions as well, which + enable or suppress certain warnings when warnings are enabled. + + All generated warnings are added to the Kconfig.warnings list. See + the class documentation. + + warn_to_stderr (default: True): + True if warnings should be printed to stderr in addition to being + added to Kconfig.warnings. + + This can be changed later with + Kconfig.enable/disable_stderr_warnings(). + + encoding (default: "utf-8"): + The encoding to use when reading and writing files. If None, the + encoding specified in the current locale will be used. + + The "utf-8" default avoids exceptions on systems that are configured + to use the C locale, which implies an ASCII encoding. + + This parameter has no effect on Python 2, due to implementation + issues (regular strings turning into Unicode strings, which are + distinct in Python 2). Python 2 doesn't decode regular strings + anyway. + + Related PEP: https://www.python.org/dev/peps/pep-0538/ + """ + self.srctree = os.environ.get("srctree", "") + self.config_prefix = os.environ.get("CONFIG_", "CONFIG_") + + # Regular expressions for parsing .config files + self._set_match = _re_match(self.config_prefix + r"([^=]+)=(.*)") + self._unset_match = \ + _re_match(r"# {}([^ ]+) is not set".format(self.config_prefix)) + + + self.warnings = [] + + self._warnings_enabled = warn + self._warn_to_stderr = warn_to_stderr + self._warn_for_undef_assign = False + self._warn_for_redun_assign = True + + + self._encoding = encoding + + + self.syms = {} + self.const_syms = {} + self.defined_syms = [] + + self.named_choices = {} + self.choices = [] + + self.menus = [] + self.comments = [] + + for nmy in "n", "m", "y": + sym = Symbol() + sym.kconfig = self + sym.name = nmy + sym.is_constant = True + sym.orig_type = TRISTATE + sym._cached_tri_val = STR_TO_TRI[nmy] + + self.const_syms[nmy] = sym + + self.n = self.const_syms["n"] + self.m = self.const_syms["m"] + self.y = self.const_syms["y"] + + # Make n/m/y well-formed symbols + for nmy in "n", "m", "y": + sym = self.const_syms[nmy] + sym.rev_dep = sym.weak_rev_dep = sym.direct_dep = self.n + + + # Maps preprocessor variables names to Variable instances + self.variables = {} + + # Predefined preprocessor functions, with min/max number of arguments + self._functions = { + "info": (_info_fn, 1, 1), + "error-if": (_error_if_fn, 2, 2), + "filename": (_filename_fn, 0, 0), + "lineno": (_lineno_fn, 0, 0), + "shell": (_shell_fn, 1, 1), + "warning-if": (_warning_if_fn, 2, 2), + } + + + # This is used to determine whether previously unseen symbols should be + # registered. They shouldn't be if we parse expressions after parsing, + # as part of Kconfig.eval_string(). + self._parsing_kconfigs = True + + self.modules = self._lookup_sym("MODULES") + self.defconfig_list = None + + self.top_node = MenuNode() + self.top_node.kconfig = self + self.top_node.item = MENU + self.top_node.is_menuconfig = True + self.top_node.visibility = self.y + self.top_node.prompt = ("Main menu", self.y) + self.top_node.parent = None + self.top_node.dep = self.y + self.top_node.filename = os.path.relpath(filename, self.srctree) + self.top_node.linenr = 1 + + # Parse the Kconfig files + + # These implement a single line of "unget" for the parser + self._saved_line = None + self._has_tokens = False + + # Keeps track of the location in the parent Kconfig files. Kconfig + # files usually source other Kconfig files. + self._filestack = [] + + # The current parsing location + self._filename = os.path.relpath(filename, self.srctree) + self._linenr = 0 + + # Open the top-level Kconfig file + self._file = self._open(filename, "r") + + try: + # Parse everything + self._parse_block(None, self.top_node, self.top_node) + except UnicodeDecodeError as e: + _decoding_error(e, self._filename) + + self.top_node.list = self.top_node.next + self.top_node.next = None + + # Projects like U-Boot and Zephyr make heavy use of being able to + # define a symbol in multiple locations. Removing duplicates makes a + # massive difference for U-Boot, speeding up parsing from ~4 seconds to + # ~0.6 seconds on my machine. + self._defined_syms_set = set(self.defined_syms) + + self._parsing_kconfigs = False + + # Do various post-processing of the menu tree + self._finalize_tree(self.top_node, self.y) + + + # Do sanity checks. Some of these depend on everything being + # finalized. + + for sym in self._defined_syms_set: + _check_sym_sanity(sym) + + for choice in self.choices: + _check_choice_sanity(choice) + + if os.environ.get("KCONFIG_STRICT") == "y": + self._check_undefined_syms() + + + # Build Symbol._dependents for all symbols and choices + self._build_dep() + + # Check for dependency loops + for sym in self._defined_syms_set: + _check_dep_loop_sym(sym, False) + + # Add extra dependencies from choices to choice symbols that get + # awkward during dependency loop detection + self._add_choice_deps() + + + self._warn_for_no_prompt = True + + self.mainmenu_text = self.top_node.prompt[0] + + @property + def defconfig_filename(self): + """ + See the class documentation. + """ + if self.defconfig_list: + for filename, cond in self.defconfig_list.defaults: + if expr_value(cond): + try: + with self._open_config(filename.str_value) as f: + return f.name + except IOError: + continue + + return None + + def load_config(self, filename, replace=True): + """ + Loads symbol values from a file in the .config format. Equivalent to + calling Symbol.set_value() to set each of the values. + + "# CONFIG_FOO is not set" within a .config file sets the user value of + FOO to n. The C tools work the same way. + + The Symbol.user_value attribute can be inspected afterwards to see what + value the symbol was assigned in the .config file (if any). The user + value might differ from Symbol.str/tri_value if there are unsatisfied + dependencies. + + filename: + The file to load. Respects $srctree if set (see the class + documentation). + + replace (default: True): + True if all existing user values should be cleared before loading the + .config. + """ + # Disable the warning about assigning to symbols without prompts. This + # is normal and expected within a .config file. + self._warn_for_no_prompt = False + + # This stub only exists to make sure _warn_for_no_prompt gets reenabled + try: + self._load_config(filename, replace) + except UnicodeDecodeError as e: + _decoding_error(e, filename) + finally: + self._warn_for_no_prompt = True + + def _load_config(self, filename, replace): + with self._open_config(filename) as f: + if replace: + # If we're replacing the configuration, keep track of which + # symbols and choices got set so that we can unset the rest + # later. This avoids invalidating everything and is faster. + # Another benefit is that invalidation must be rock solid for + # it to work, making it a good test. + + for sym in self._defined_syms_set: + sym._was_set = False + + for choice in self.choices: + choice._was_set = False + + # Small optimizations + set_match = self._set_match + unset_match = self._unset_match + syms = self.syms + + for linenr, line in enumerate(f, 1): + # The C tools ignore trailing whitespace + line = line.rstrip() + + match = set_match(line) + if match: + name, val = match.groups() + if name not in syms: + self._warn_undef_assign_load(name, val, filename, + linenr) + continue + + sym = syms[name] + if not sym.nodes: + self._warn_undef_assign_load(name, val, filename, + linenr) + continue + + if sym.orig_type in (BOOL, TRISTATE): + # The C implementation only checks the first character + # to the right of '=', for whatever reason + if not ((sym.orig_type == BOOL and + val.startswith(("n", "y"))) or \ + (sym.orig_type == TRISTATE and + val.startswith(("n", "m", "y")))): + self._warn("'{}' is not a valid value for the {} " + "symbol {}. Assignment ignored." + .format(val, TYPE_TO_STR[sym.orig_type], + _name_and_loc(sym)), + filename, linenr) + continue + + val = val[0] + + if sym.choice and val != "n": + # During .config loading, we infer the mode of the + # choice from the kind of values that are assigned + # to the choice symbols + + prev_mode = sym.choice.user_value + if prev_mode is not None and \ + TRI_TO_STR[prev_mode] != val: + + self._warn("both m and y assigned to symbols " + "within the same choice", + filename, linenr) + + # Set the choice's mode + sym.choice.set_value(val) + + elif sym.orig_type == STRING: + match = _conf_string_match(val) + if not match: + self._warn("malformed string literal in " + "assignment to {}. Assignment ignored." + .format(_name_and_loc(sym)), + filename, linenr) + continue + + val = unescape(match.group(1)) + + else: + match = unset_match(line) + if not match: + # Print a warning for lines that match neither + # set_match() nor unset_match() and that are not blank + # lines or comments. 'line' has already been + # rstrip()'d, so blank lines show up as "" here. + if line and not line.lstrip().startswith("#"): + self._warn("ignoring malformed line '{}'" + .format(line), + filename, linenr) + + continue + + name = match.group(1) + if name not in syms: + self._warn_undef_assign_load(name, "n", filename, + linenr) + continue + + sym = syms[name] + if sym.orig_type not in (BOOL, TRISTATE): + continue + + val = "n" + + # Done parsing the assignment. Set the value. + + if sym._was_set: + # Use strings for bool/tristate user values in the warning + if sym.orig_type in (BOOL, TRISTATE): + display_user_val = TRI_TO_STR[sym.user_value] + else: + display_user_val = sym.user_value + + warn_msg = '{} set more than once. Old value: "{}", new value: "{}".'.format( + _name_and_loc(sym), display_user_val, val + ) + + if display_user_val == val: + self._warn_redun_assign(warn_msg, filename, linenr) + else: + self._warn( warn_msg, filename, linenr) + + sym.set_value(val) + + if replace: + # If we're replacing the configuration, unset the symbols that + # didn't get set + + for sym in self._defined_syms_set: + if not sym._was_set: + sym.unset_value() + + for choice in self.choices: + if not choice._was_set: + choice.unset_value() + + def write_autoconf(self, filename, + header="/* Generated by Kconfiglib (https://github.com/ulfalizer/Kconfiglib) */\n"): + r""" + Writes out symbol values as a C header file, matching the format used + by include/generated/autoconf.h in the kernel. + + The ordering of the #defines matches the one generated by + write_config(). The order in the C implementation depends on the hash + table implementation as of writing, and so won't match. + + filename: + Self-explanatory. + + header (default: "/* Generated by Kconfiglib (https://github.com/ulfalizer/Kconfiglib) */\n"): + Text that will be inserted verbatim at the beginning of the file. You + would usually want it enclosed in '/* */' to make it a C comment, + and include a final terminating newline. + """ + with self._open(filename, "w") as f: + f.write(header) + + # Avoid duplicates -- see write_config() + for sym in self._defined_syms_set: + sym._written = False + + for sym in self._defined_syms_set: + if not sym._written: + sym._written = True + # Note: _write_to_conf is determined when the value is + # calculated. This is a hidden function call due to + # property magic. + val = sym.str_value + if sym._write_to_conf: + if sym.orig_type in (BOOL, TRISTATE): + if val != "n": + f.write("#define {}{}{} 1\n" + .format(self.config_prefix, sym.name, + "_MODULE" if val == "m" else "")) + + elif sym.orig_type == STRING: + f.write('#define {}{} "{}"\n' + .format(self.config_prefix, sym.name, + escape(val))) + + elif sym.orig_type in (INT, HEX): + if sym.orig_type == HEX and \ + not val.startswith(("0x", "0X")): + val = "0x" + val + + f.write("#define {}{} {}\n" + .format(self.config_prefix, sym.name, val)) + + else: + _internal_error("Internal error while creating C " + 'header: unknown type "{}".' + .format(sym.orig_type)) + + def write_config(self, filename, + header="# Generated by Kconfiglib (https://github.com/ulfalizer/Kconfiglib)\n"): + r""" + Writes out symbol values in the .config format. The format matches the + C implementation, including ordering. + + Symbols appear in the same order in generated .config files as they do + in the Kconfig files. For symbols defined in multiple locations, a + single assignment is written out corresponding to the first location + where the symbol is defined. + + See the 'Intro to symbol values' section in the module docstring to + understand which symbols get written out. + + filename: + Self-explanatory. + + header (default: "# Generated by Kconfiglib (https://github.com/ulfalizer/Kconfiglib)\n"): + Text that will be inserted verbatim at the beginning of the file. You + would usually want each line to start with '#' to make it a comment, + and include a final terminating newline. + """ + with self._open(filename, "w") as f: + f.write(header) + + # Symbol._written is set to True when a symbol config string is + # fetched, so that symbols defined in multiple locations only get + # one .config entry. We reset it prior to writing out a new + # .config. It only needs to be reset for defined symbols, because + # undefined symbols will never be written out (because they do not + # appear in the menu tree rooted at Kconfig.top_node). + # + # The C tools reuse _write_to_conf for this, but we cache + # _write_to_conf together with the value and don't invalidate + # cached values when writing .config files, so that won't work. + for sym in self._defined_syms_set: + sym._written = False + + node = self.top_node.list + if not node: + # Empty configuration + return + + while 1: + item = node.item + if isinstance(item, Symbol): + if not item._written: + item._written = True + f.write(item.config_string) + + elif expr_value(node.dep) and \ + ((item == MENU and expr_value(node.visibility)) or + item == COMMENT): + + f.write("\n#\n# {}\n#\n".format(node.prompt[0])) + + # Iterative tree walk using parent pointers + + if node.list: + node = node.list + elif node.next: + node = node.next + else: + while node.parent: + node = node.parent + if node.next: + node = node.next + break + else: + return + + def write_min_config(self, filename, + header="# Generated by Kconfiglib (https://github.com/ulfalizer/Kconfiglib)\n"): + """ + Writes out a "minimal" configuration file, omitting symbols whose value + matches their default value. The format matches the one produced by + 'make savedefconfig'. + + The resulting configuration file is incomplete, but a complete + configuration can be derived from it by loading it. Minimal + configuration files can serve as a more manageable configuration format + compared to a "full" .config file, especially when configurations files + are merged or edited by hand. + + filename: + Self-explanatory. + + header (default: "# Generated by Kconfiglib (https://github.com/ulfalizer/Kconfiglib)\n"): + Text that will be inserted verbatim at the beginning of the file. You + would usually want each line to start with '#' to make it a comment, + and include a final terminating newline. + """ + with self._open(filename, "w") as f: + f.write(header) + + # Avoid duplicates -- see write_config() + for sym in self._defined_syms_set: + sym._written = False + + for sym in self._defined_syms_set: + if not sym._written: + sym._written = True + + # Skip symbols that cannot be changed. Only check + # non-choice symbols, as selects don't affect choice + # symbols. + if not sym.choice and \ + sym.visibility <= expr_value(sym.rev_dep): + continue + + # Skip symbols whose value matches their default + if sym.str_value == sym._str_default(): + continue + + # Skip symbols that would be selected by default in a + # choice, unless the choice is optional or the symbol type + # isn't bool (it might be possible to set the choice mode + # to n or the symbol to m in those cases). + if sym.choice and \ + not sym.choice.is_optional and \ + sym.choice._get_selection_from_defaults() is sym and \ + sym.orig_type == BOOL and \ + sym.tri_value == 2: + continue + + f.write(sym.config_string) + + def sync_deps(self, path): + """ + Creates or updates a directory structure that can be used to avoid + doing a full rebuild whenever the configuration is changed, mirroring + include/config/ in the kernel. + + This function is intended to be called during each build, before + compiling source files that depend on configuration symbols. + + path: + Path to directory + + sync_deps(path) does the following: + + 1. If the directory <path> does not exist, it is created. + + 2. If <path>/auto.conf exists, old symbol values are loaded from it, + which are then compared against the current symbol values. If a + symbol has changed value (would generate different output in + autoconf.h compared to before), the change is signaled by + touch'ing a file corresponding to the symbol. + + The first time sync_deps() is run on a directory, <path>/auto.conf + won't exist, and no old symbol values will be available. This + logically has the same effect as updating the entire + configuration. + + The path to a symbol's file is calculated from the symbol's name + by replacing all '_' with '/' and appending '.h'. For example, the + symbol FOO_BAR_BAZ gets the file <path>/foo/bar/baz.h, and FOO + gets the file <path>/foo.h. + + This scheme matches the C tools. The point is to avoid having a + single directory with a huge number of files, which the underlying + filesystem might not handle well. + + 3. A new auto.conf with the current symbol values is written, to keep + track of them for the next build. + + + The last piece of the puzzle is knowing what symbols each source file + depends on. Knowing that, dependencies can be added from source files + to the files corresponding to the symbols they depends on. The source + file will then get recompiled (only) when the symbol value changes + (provided sync_deps() is run first during each build). + + The tool in the kernel that extracts symbol dependencies from source + files is scripts/basic/fixdep.c. Missing symbol files also correspond + to "not changed", which fixdep deals with by using the $(wildcard) Make + function when adding symbol prerequisites to source files. + + In case you need a different scheme for your project, the sync_deps() + implementation can be used as a template.""" + if not os.path.exists(path): + os.mkdir(path, 0o755) + + # This setup makes sure that at least the current working directory + # gets reset if things fail + prev_dir = os.getcwd() + try: + # cd'ing into the symbol file directory simplifies + # _sync_deps() and saves some work + os.chdir(path) + self._sync_deps() + finally: + os.chdir(prev_dir) + + def _sync_deps(self): + # Load old values from auto.conf, if any + self._load_old_vals() + + for sym in self._defined_syms_set: + # Note: _write_to_conf is determined when the value is + # calculated. This is a hidden function call due to + # property magic. + val = sym.str_value + + # Note: n tristate values do not get written to auto.conf and + # autoconf.h, making a missing symbol logically equivalent to n + + if sym._write_to_conf: + if sym._old_val is None and \ + sym.orig_type in (BOOL, TRISTATE) and \ + val == "n": + # No old value (the symbol was missing or n), new value n. + # No change. + continue + + if val == sym._old_val: + # New value matches old. No change. + continue + + elif sym._old_val is None: + # The symbol wouldn't appear in autoconf.h (because + # _write_to_conf is false), and it wouldn't have appeared in + # autoconf.h previously either (because it didn't appear in + # auto.conf). No change. + continue + + # 'sym' has a new value. Flag it. + + sym_path = sym.name.lower().replace("_", os.sep) + ".h" + sym_path_dir = os.path.dirname(sym_path) + if sym_path_dir and not os.path.exists(sym_path_dir): + os.makedirs(sym_path_dir, 0o755) + + # A kind of truncating touch, mirroring the C tools + os.close(os.open( + sym_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644)) + + # Remember the current values as the "new old" values. + # + # This call could go anywhere after the call to _load_old_vals(), but + # putting it last means _sync_deps() can be safely rerun if it fails + # before this point. + self._write_old_vals() + + def _write_old_vals(self): + # Helper for writing auto.conf. Basically just a simplified + # write_config() that doesn't write any comments (including + # '# CONFIG_FOO is not set' comments). The format matches the C + # implementation, though the ordering is arbitrary there (depends on + # the hash table implementation). + # + # A separate helper function is neater than complicating write_config() + # by passing a flag to it, plus we only need to look at symbols here. + + with self._open("auto.conf", "w") as f: + for sym in self._defined_syms_set: + sym._written = False + + for sym in self._defined_syms_set: + if not sym._written: + sym._written = True + if not (sym.orig_type in (BOOL, TRISTATE) and + not sym.tri_value): + f.write(sym.config_string) + + def _load_old_vals(self): + # Loads old symbol values from auto.conf into a dedicated + # Symbol._old_val field. Mirrors load_config(). + # + # The extra field could be avoided with some trickery involving dumping + # symbol values and restoring them later, but this is simpler and + # faster. The C tools also use a dedicated field for this purpose. + + for sym in self._defined_syms_set: + sym._old_val = None + + if not os.path.exists("auto.conf"): + # No old values + return + + with self._open("auto.conf", "r") as f: + for line in f: + match = self._set_match(line) + if not match: + # We only expect CONFIG_FOO=... (and possibly a header + # comment) in auto.conf + continue + + name, val = match.groups() + if name in self.syms: + sym = self.syms[name] + + if sym.orig_type == STRING: + match = _conf_string_match(val) + if not match: + continue + val = unescape(match.group(1)) + + self.syms[name]._old_val = val + + def eval_string(self, s): + """ + Returns the tristate value of the expression 's', represented as 0, 1, + and 2 for n, m, and y, respectively. Raises KconfigError if syntax + errors are detected in 's'. Warns if undefined symbols are referenced. + + As an example, if FOO and BAR are tristate symbols at least one of + which has the value y, then config.eval_string("y && (FOO || BAR)") + returns 2 (y). + + To get the string value of non-bool/tristate symbols, use + Symbol.str_value. eval_string() always returns a tristate value, and + all non-bool/tristate symbols have the tristate value 0 (n). + + The expression parsing is consistent with how parsing works for + conditional ('if ...') expressions in the configuration, and matches + the C implementation. m is rewritten to 'm && MODULES', so + eval_string("m") will return 0 (n) unless modules are enabled. + """ + # The parser is optimized to be fast when parsing Kconfig files (where + # an expression can never appear at the beginning of a line). We have + # to monkey-patch things a bit here to reuse it. + + self._filename = None + + # Don't include the "if " from below to avoid giving confusing error + # messages + self._line = s + # Remove the _T_IF token + self._tokens = self._tokenize("if " + s)[1:] + self._tokens_i = -1 + + return expr_value(self._parse_expr(True)) # transform_m + + def unset_values(self): + """ + Resets the user values of all symbols, as if Kconfig.load_config() or + Symbol.set_value() had never been called. + """ + self._warn_for_no_prompt = False + try: + # set_value() already rejects undefined symbols, and they don't + # need to be invalidated (because their value never changes), so we + # can just iterate over defined symbols + for sym in self._defined_syms_set: + sym.unset_value() + + for choice in self.choices: + choice.unset_value() + finally: + self._warn_for_no_prompt = True + + def enable_warnings(self): + """ + See Kconfig.__init__(). + """ + self._warnings_enabled = True + + def disable_warnings(self): + """ + See Kconfig.__init__(). + """ + self._warnings_enabled = False + + def enable_stderr_warnings(self): + """ + See Kconfig.__init__(). + """ + self._warn_to_stderr = True + + def disable_stderr_warnings(self): + """ + See Kconfig.__init__(). + """ + self._warn_to_stderr = False + + def enable_undef_warnings(self): + """ + Enables warnings for assignments to undefined symbols. Disabled by + default since they tend to be spammy for Kernel configurations (and + mostly suggests cleanups). + """ + self._warn_for_undef_assign = True + + def disable_undef_warnings(self): + """ + See enable_undef_assign(). + """ + self._warn_for_undef_assign = False + + def enable_redun_warnings(self): + """ + Enables warnings for duplicated assignments in .config files that all + set the same value. + + These warnings are enabled by default. Disabling them might be helpful + in certain cases when merging configurations. + """ + self._warn_for_redun_assign = True + + def disable_redun_warnings(self): + """ + See enable_redun_warnings(). + """ + self._warn_for_redun_assign = False + + def __repr__(self): + """ + Returns a string with information about the Kconfig object when it is + evaluated on e.g. the interactive Python prompt. + """ + return "<{}>".format(", ".join(( + "configuration with {} symbols".format(len(self.syms)), + 'main menu prompt "{}"'.format(self.mainmenu_text), + "srctree is current directory" if not self.srctree else + 'srctree "{}"'.format(self.srctree), + 'config symbol prefix "{}"'.format(self.config_prefix), + "warnings " + + ("enabled" if self._warnings_enabled else "disabled"), + "printing of warnings to stderr " + + ("enabled" if self._warn_to_stderr else "disabled"), + "undef. symbol assignment warnings " + + ("enabled" if self._warn_for_undef_assign else "disabled"), + "redundant symbol assignment warnings " + + ("enabled" if self._warn_for_redun_assign else "disabled") + ))) + + # + # Private methods + # + + + # + # File reading + # + + def _open_config(self, filename): + # Opens a .config file. First tries to open 'filename', then + # '$srctree/filename' if $srctree was set when the configuration was + # loaded. + + try: + return self._open(filename, "r") + except IOError as e: + # This will try opening the same file twice if $srctree is unset, + # but it's not a big deal + try: + return self._open(os.path.join(self.srctree, filename), "r") + except IOError as e2: + # This is needed for Python 3, because e2 is deleted after + # the try block: + # + # https://docs.python.org/3/reference/compound_stmts.html#the-try-statement + e = e2 + + raise IOError("\n" + textwrap.fill( + "Could not open '{}' ({}: {}){}".format( + filename, errno.errorcode[e.errno], e.strerror, + self._srctree_hint()), + 80)) + + def _enter_file(self, full_filename, rel_filename): + # Jumps to the beginning of a sourced Kconfig file, saving the previous + # position and file object. + # + # full_filename: + # Actual path to the file. + # + # rel_filename: + # File path with $srctree prefix stripped, stored in e.g. + # self._filename (which makes it indirectly show up in + # MenuNode.filename). Equals full_filename for absolute paths. + + self._filestack.append((self._filename, self._linenr, self._file)) + + # Check for recursive 'source' + for name, _, _ in self._filestack: + if name == rel_filename: + raise KconfigError( + "\n{}:{}: Recursive 'source' of '{}' detected. Check that " + "environment variables are set correctly.\n" + "Backtrace:\n{}" + .format(self._filename, self._linenr, filename, + "\n".join("{}:{}".format(name, linenr) + for name, linenr, _ + in reversed(self._filestack)))) + + # Note: We already know that the file exists + + try: + self._file = self._open(full_filename, "r") + except IOError as e: + raise IOError("{}:{}: Could not open '{}' ({}: {})".format( + self._filename, self._linenr, full_filename, + errno.errorcode[e.errno], e.strerror)) + + self._filename = rel_filename + self._linenr = 0 + + def _leave_file(self): + # Returns from a Kconfig file to the file that sourced it + + self._file.close() + self._filename, self._linenr, self._file = self._filestack.pop() + + def _next_line(self): + # Fetches and tokenizes the next line from the current Kconfig file. + # Returns False at EOF and True otherwise. + + # _saved_line provides a single line of "unget", currently only used + # for help texts. + # + # This also works as expected if _saved_line is "", indicating EOF: + # "" is falsy, and readline() returns "" over and over at EOF. + if self._saved_line: + self._line = self._saved_line + self._saved_line = None + else: + self._line = self._file.readline() + if not self._line: + return False + self._linenr += 1 + + # Handle line joining + while self._line.endswith("\\\n"): + self._line = self._line[:-2] + self._file.readline() + self._linenr += 1 + + self._tokens = self._tokenize(self._line) + self._tokens_i = -1 # Token index (minus one) + + return True + + + # + # Tokenization + # + + def _lookup_sym(self, name): + # Fetches the symbol 'name' from the symbol table, creating and + # registering it if it does not exist. If '_parsing_kconfigs' is False, + # it means we're in eval_string(), and new symbols won't be registered. + + if name in self.syms: + return self.syms[name] + + sym = Symbol() + sym.kconfig = self + sym.name = name + sym.is_constant = False + sym.rev_dep = sym.weak_rev_dep = sym.direct_dep = self.n + + if self._parsing_kconfigs: + self.syms[name] = sym + else: + self._warn("no symbol {} in configuration".format(name)) + + return sym + + def _lookup_const_sym(self, name): + # Like _lookup_sym(), for constant (quoted) symbols + + if name in self.const_syms: + return self.const_syms[name] + + sym = Symbol() + sym.kconfig = self + sym.name = name + sym.is_constant = True + sym.rev_dep = sym.weak_rev_dep = sym.direct_dep = self.n + + if self._parsing_kconfigs: + self.const_syms[name] = sym + + return sym + + def _tokenize(self, s): + # Parses 's', returning a None-terminated list of tokens. Registers any + # new symbols encountered with _lookup(_const)_sym(). + # + # Tries to be reasonably speedy by processing chunks of text via + # regexes and string operations where possible. This is the biggest + # hotspot during parsing. + # + # Note: It might be possible to rewrite this to 'yield' tokens instead, + # working across multiple lines. The 'option env' lookback thing below + # complicates things though. + + # Initial token on the line + match = _command_match(s) + if not match: + if s.isspace() or s.lstrip().startswith("#"): + return (None,) + self._parse_error("unknown token at start of line") + + # Tricky implementation detail: While parsing a token, 'token' refers + # to the previous token. See _STRING_LEX for why this is needed. + token = _get_keyword(match.group(1)) + if not token: + # Backwards compatibility with old versions of the C tools, which + # (accidentally) accepted stuff like "--help--" and "-help---". + # This was fixed in the C tools by commit c2264564 ("kconfig: warn + # of unhandled characters in Kconfig commands"), committed in July + # 2015, but it seems people still run Kconfiglib on older kernels. + if s.strip(" \t\n-") == "help": + return (_T_HELP, None) + + # If the first token is not a keyword (and not a weird help token), + # we have a preprocessor variable assignment (or a bare macro on a + # line) + self._parse_assignment(s) + return (None,) + + tokens = [token] + # The current index in the string being tokenized + i = match.end() + + # Main tokenization loop (for tokens past the first one) + while i < len(s): + # Test for an identifier/keyword first. This is the most common + # case. + match = _id_keyword_match(s, i) + if match: + # We have an identifier or keyword + + # Jump past it + i = match.end() + + # Check what it is. lookup_sym() will take care of allocating + # new symbols for us the first time we see them. Note that + # 'token' still refers to the previous token. + + name = match.group(1) + keyword = _get_keyword(name) + if keyword: + # It's a keyword + token = keyword + + elif token not in _STRING_LEX: + # It's a non-const symbol, except we translate n, m, and y + # into the corresponding constant symbols, like the C + # implementation + token = self.const_syms[name] \ + if name in ("n", "m", "y") else \ + self._lookup_sym(name) + + else: + # It's a case of missing quotes. For example, the + # following is accepted: + # + # menu unquoted_title + # + # config A + # tristate unquoted_prompt + # + # endmenu + token = name + + else: + # Neither a keyword nor a non-const symbol (except + # $()-expansion might still yield a non-const symbol). + + # We always strip whitespace after tokens, so it is safe to + # assume that s[i] is the start of a token here. + c = s[i] + + if c in "\"'": + s, end_i = self._expand_str(s, i, c) + + # os.path.expandvars() and the $UNAME_RELEASE replace() is + # a backwards compatibility hack, which should be + # reasonably safe as expandvars() leaves references to + # undefined env. vars. as is. + # + # The preprocessor functionality changed how environment + # variables are referenced, to $(FOO). + val = os.path.expandvars( + s[i + 1:end_i - 1].replace("$UNAME_RELEASE", + platform.uname()[2])) + + i = end_i + + # This is the only place where we don't survive with a + # single token of lookback: 'option env="FOO"' does not + # refer to a constant symbol named "FOO". + token = val \ + if token in _STRING_LEX or \ + tokens[0] == _T_OPTION else \ + self._lookup_const_sym(val) + + elif s.startswith("&&", i): + token = _T_AND + i += 2 + + elif s.startswith("||", i): + token = _T_OR + i += 2 + + elif c == "=": + token = _T_EQUAL + i += 1 + + elif s.startswith("!=", i): + token = _T_UNEQUAL + i += 2 + + elif c == "!": + token = _T_NOT + i += 1 + + elif c == "(": + token = _T_OPEN_PAREN + i += 1 + + elif c == ")": + token = _T_CLOSE_PAREN + i += 1 + + elif s.startswith("$(", i): + s, end_i = self._expand_macro(s, i, ()) + val = s[i:end_i] + # isspace() is False for empty strings + if not val.strip(): + # Avoid creating a Kconfig symbol with a blank name. + # It's almost guaranteed to be an error. + self._parse_error("macro expanded to blank string") + i = end_i + + # Compatibility with what the C implementation does. Might + # be unexpected that you can reference non-constant symbols + # this way though... + token = self.const_syms[val] \ + if val in ("n", "m", "y") else \ + self._lookup_sym(val) + + elif c == "#": + break + + + # Very rare + + elif s.startswith("<=", i): + token = _T_LESS_EQUAL + i += 2 + + elif c == "<": + token = _T_LESS + i += 1 + + elif s.startswith(">=", i): + token = _T_GREATER_EQUAL + i += 2 + + elif c == ">": + token = _T_GREATER + i += 1 + + + else: + self._parse_error("unknown tokens in line") + + + # Skip trailing whitespace + while i < len(s) and s[i].isspace(): + i += 1 + + + # Add the token + tokens.append(token) + + # None-terminating the token list makes the token fetching functions + # simpler/faster + tokens.append(None) + + return tokens + + def _next_token(self): + self._tokens_i += 1 + return self._tokens[self._tokens_i] + + def _peek_token(self): + return self._tokens[self._tokens_i + 1] + + # The functions below are just _next_token() with extra syntax checking. + # Inlining _next_token() and _peek_token() into them saves a few % of + # parsing time. + # + # See the 'Intro to expressions' section for what a constant symbol is. + + def _expect_sym(self): + self._tokens_i += 1 + token = self._tokens[self._tokens_i] + + if not isinstance(token, Symbol): + self._parse_error("expected symbol") + + return token + + def _expect_nonconst_sym(self): + self._tokens_i += 1 + token = self._tokens[self._tokens_i] + + if not isinstance(token, Symbol) or token.is_constant: + self._parse_error("expected nonconstant symbol") + + return token + + def _expect_nonconst_sym_and_eol(self): + self._tokens_i += 1 + token = self._tokens[self._tokens_i] + + if not isinstance(token, Symbol) or token.is_constant: + self._parse_error("expected nonconstant symbol") + + if self._tokens[self._tokens_i + 1] is not None: + self._parse_error("extra tokens at end of line") + + return token + + def _expect_str(self): + self._tokens_i += 1 + token = self._tokens[self._tokens_i] + + if not isinstance(token, str): + self._parse_error("expected string") + + return token + + def _expect_str_and_eol(self): + self._tokens_i += 1 + token = self._tokens[self._tokens_i] + + if not isinstance(token, str): + self._parse_error("expected string") + + if self._tokens[self._tokens_i + 1] is not None: + self._parse_error("extra tokens at end of line") + + return token + + def _check_token(self, token): + # If the next token is 'token', removes it and returns True + + if self._tokens[self._tokens_i + 1] == token: + self._tokens_i += 1 + return True + return False + + + # + # Preprocessor logic + # + + def _parse_assignment(self, s): + # Parses a preprocessor variable assignment, registering the variable + # if it doesn't already exist. Also takes care of bare macros on lines + # (which are allowed, and can be useful for their side effects). + + # Expand any macros in the left-hand side of the assignment (the + # variable name) + s = s.lstrip() + i = 0 + while 1: + i = _assignment_lhs_fragment_match(s, i).end() + if s.startswith("$(", i): + s, i = self._expand_macro(s, i, ()) + else: + break + + if s.isspace(): + # We also accept a bare macro on a line (e.g. + # $(warning-if,$(foo),ops)), provided it expands to a blank string + return + + # Assigned variable + name = s[:i] + + + # Extract assignment operator (=, :=, or +=) and value + rhs_match = _assignment_rhs_match(s, i) + if not rhs_match: + self._parse_error("syntax error") + + op, val = rhs_match.groups() + + + if name in self.variables: + # Already seen variable + var = self.variables[name] + else: + # New variable + var = Variable() + var.kconfig = self + var.name = name + var._n_expansions = 0 + self.variables[name] = var + + # += acts like = on undefined variables (defines a recursive + # variable) + if op == "+=": + op = "=" + + if op == "=": + var.is_recursive = True + var.value = val + elif op == ":=": + var.is_recursive = False + var.value = self._expand_whole(val, ()) + else: # op == "+=" + # += does immediate expansion if the variable was last set + # with := + var.value += " " + (val if var.is_recursive else \ + self._expand_whole(val, ())) + + def _expand_whole(self, s, args): + # Expands preprocessor macros in all of 's'. Used whenever we don't + # have to worry about delimiters. See _expand_macro() re. the 'args' + # parameter. + # + # Returns the expanded string. + + i = 0 + while 1: + i = s.find("$(", i) + if i == -1: + break + s, i = self._expand_macro(s, i, args) + return s + + def _expand_str(self, s, i, quote): + # Expands a quoted string starting at index 'i' in 's'. Handles both + # backslash escapes and macro expansion. + # + # Returns the expanded 's' (including the part before the string) and + # the index of the first character after the expanded string in 's'. + + i += 1 # Skip over initial "/' + while 1: + match = _string_special_search(s, i) + if not match: + self._parse_error("unterminated string") + + + if match.group() == quote: + # Found the end of the string + return (s, match.end()) + + elif match.group() == "\\": + # Replace '\x' with 'x'. 'i' ends up pointing to the character + # after 'x', which allows macros to be canceled with '\$(foo)'. + i = match.end() + s = s[:match.start()] + s[i:] + + elif match.group() == "$(": + # A macro call within the string + s, i = self._expand_macro(s, match.start(), ()) + + else: + # A ' quote within " quotes or vice versa + i += 1 + + def _expand_macro(self, s, i, args): + # Expands a macro starting at index 'i' in 's'. If this macro resulted + # from the expansion of another macro, 'args' holds the arguments + # passed to that macro. + # + # Returns the expanded 's' (including the part before the macro) and + # the index of the first character after the expanded macro in 's'. + + start = i + i += 2 # Skip over "$(" + + # Start of current macro argument + arg_start = i + + # Arguments of this macro call + new_args = [] + + while 1: + match = _macro_special_search(s, i) + if not match: + self._parse_error("missing end parenthesis in macro expansion") + + + if match.group() == ")": + # Found the end of the macro + + new_args.append(s[arg_start:match.start()]) + + prefix = s[:start] + + # $(1) is replaced by the first argument to the function, etc., + # provided at least that many arguments were passed + + try: + # Does the macro look like an integer, with a corresponding + # argument? If so, expand it to the value of the argument. + prefix += args[int(new_args[0])] + except (ValueError, IndexError): + # Regular variables are just functions without arguments, + # and also go through the function value path + prefix += self._fn_val(new_args) + + return (prefix + s[match.end():], + len(prefix)) + + elif match.group() == ",": + # Found the end of a macro argument + new_args.append(s[arg_start:match.start()]) + arg_start = i = match.end() + + else: # match.group() == "$(" + # A nested macro call within the macro + s, i = self._expand_macro(s, match.start(), args) + + def _fn_val(self, args): + # Returns the result of calling the function args[0] with the arguments + # args[1..len(args)-1]. Plain variables are treated as functions + # without arguments. + + fn = args[0] + + if fn in self.variables: + var = self.variables[fn] + + if len(args) == 1: + # Plain variable + if var._n_expansions: + self._parse_error("Preprocessor variable {} recursively " + "references itself".format(var.name)) + elif var._n_expansions > 100: + # Allow functions to call themselves, but guess that functions + # that are overly recursive are stuck + self._parse_error("Preprocessor function {} seems stuck " + "in infinite recursion".format(var.name)) + + var._n_expansions += 1 + res = self._expand_whole(self.variables[fn].value, args) + var._n_expansions -= 1 + return res + + if fn in self._functions: + # Built-in function + + py_fn, min_arg, max_arg = self._functions[fn] + + if not min_arg <= len(args) - 1 <= max_arg: + if min_arg == max_arg: + expected_args = min_arg + else: + expected_args = "{}-{}".format(min_arg, max_arg) + + raise KconfigError("{}:{}: bad number of arguments in call " + "to {}, expected {}, got {}" + .format(self._filename, self._linenr, fn, + expected_args, len(args) - 1)) + + return py_fn(self, args) + + # Environment variables are tried last + if fn in os.environ: + return os.environ[fn] + + return "" + + + # + # Parsing + # + + def _make_and(self, e1, e2): + # Constructs an AND (&&) expression. Performs trivial simplification. + + if e1 is self.y: + return e2 + + if e2 is self.y: + return e1 + + if e1 is self.n or e2 is self.n: + return self.n + + return (AND, e1, e2) + + def _make_or(self, e1, e2): + # Constructs an OR (||) expression. Performs trivial simplification. + + if e1 is self.n: + return e2 + + if e2 is self.n: + return e1 + + if e1 is self.y or e2 is self.y: + return self.y + + return (OR, e1, e2) + + def _parse_block(self, end_token, parent, prev): + # Parses a block, which is the contents of either a file or an if, + # menu, or choice statement. + # + # end_token: + # The token that ends the block, e.g. _T_ENDIF ("endif") for ifs. + # None for files. + # + # parent: + # The parent menu node, corresponding to a menu, Choice, or 'if'. + # 'if's are flattened after parsing. + # + # prev: + # The previous menu node. New nodes will be added after this one (by + # modifying their 'next' pointer). + # + # 'prev' is reused to parse a list of child menu nodes (for a menu or + # Choice): After parsing the children, the 'next' pointer is assigned + # to the 'list' pointer to "tilt up" the children above the node. + # + # Returns the final menu node in the block (or 'prev' if the block is + # empty). This allows chaining. + + # We might already have tokens from parsing a line to check if it's a + # property and discovering it isn't. self._has_tokens functions as a + # kind of "unget". + while self._has_tokens or self._next_line(): + self._has_tokens = False + + t0 = self._next_token() + if t0 is None: + continue + + if t0 in (_T_CONFIG, _T_MENUCONFIG): + # The tokenizer allocates Symbol objects for us + sym = self._expect_nonconst_sym_and_eol() + self.defined_syms.append(sym) + + node = MenuNode() + node.kconfig = self + node.item = sym + node.is_menuconfig = (t0 == _T_MENUCONFIG) + node.prompt = node.help = node.list = None + node.parent = parent + node.filename = self._filename + node.linenr = self._linenr + + sym.nodes.append(node) + + self._parse_properties(node) + + if node.is_menuconfig and not node.prompt: + self._warn("the menuconfig symbol {} has no prompt" + .format(_name_and_loc(node.item))) + + # Tricky Python semantics: This assign prev.next before prev + prev.next = prev = node + + elif t0 in (_T_SOURCE, _T_RSOURCE, _T_OSOURCE, _T_ORSOURCE): + pattern = self._expect_str_and_eol() + + # Check if the pattern is absolute and avoid stripping srctree + # from it below in that case. We must do the check before + # join()'ing, as srctree might be an absolute path. + isabs = os.path.isabs(pattern) + + if t0 in (_T_RSOURCE, _T_ORSOURCE): + # Relative source + pattern = os.path.join(os.path.dirname(self._filename), + pattern) + + # Sort the glob results to ensure a consistent ordering of + # Kconfig symbols, which indirectly ensures a consistent + # ordering in e.g. .config files + filenames = \ + sorted(glob.iglob(os.path.join(self.srctree, pattern))) + + if not filenames and t0 in (_T_SOURCE, _T_RSOURCE): + raise KconfigError("\n" + textwrap.fill( + "{}:{}: '{}' does not exist{}".format( + self._filename, self._linenr, pattern, + self._srctree_hint()), + 80)) + + for filename in filenames: + self._enter_file( + filename, + # Unless an absolute path is passed to *source, strip + # the $srctree prefix from the filename. That way it + # appears without a $srctree prefix in + # MenuNode.filename, which is nice e.g. when generating + # documentation. + filename if isabs else + os.path.relpath(filename, self.srctree)) + + prev = self._parse_block(None, parent, prev) + + self._leave_file() + + elif t0 == end_token: + # We have reached the end of the block. Terminate the final + # node and return it. + prev.next = None + return prev + + elif t0 == _T_IF: + node = MenuNode() + node.item = node.prompt = None + node.parent = parent + node.filename = self._filename + node.linenr = self._linenr + + node.dep = self._parse_expr(True) + + self._parse_block(_T_ENDIF, node, node) + node.list = node.next + + prev.next = prev = node + + elif t0 == _T_MENU: + node = MenuNode() + node.kconfig = self + node.item = MENU + node.is_menuconfig = True + node.prompt = (self._expect_str_and_eol(), self.y) + node.visibility = self.y + node.parent = parent + node.filename = self._filename + node.linenr = self._linenr + + self.menus.append(node) + + self._parse_properties(node) + self._parse_block(_T_ENDMENU, node, node) + node.list = node.next + + prev.next = prev = node + + elif t0 == _T_COMMENT: + node = MenuNode() + node.kconfig = self + node.item = COMMENT + node.is_menuconfig = False + node.prompt = (self._expect_str_and_eol(), self.y) + node.list = None + node.parent = parent + node.filename = self._filename + node.linenr = self._linenr + + self.comments.append(node) + + self._parse_properties(node) + + prev.next = prev = node + + elif t0 == _T_CHOICE: + if self._peek_token() is None: + choice = Choice() + choice.direct_dep = self.n + + self.choices.append(choice) + else: + # Named choice + name = self._expect_str_and_eol() + choice = self.named_choices.get(name) + if not choice: + choice = Choice() + choice.name = name + choice.direct_dep = self.n + + self.choices.append(choice) + self.named_choices[name] = choice + + choice.kconfig = self + + node = MenuNode() + node.kconfig = self + node.item = choice + node.is_menuconfig = True + node.prompt = node.help = None + node.parent = parent + node.filename = self._filename + node.linenr = self._linenr + + choice.nodes.append(node) + + self._parse_properties(node) + self._parse_block(_T_ENDCHOICE, node, node) + node.list = node.next + + prev.next = prev = node + + elif t0 == _T_MAINMENU: + self.top_node.prompt = (self._expect_str_and_eol(), self.y) + self.top_node.filename = self._filename + self.top_node.linenr = self._linenr + + else: + self._parse_error("unrecognized construct") + + # End of file reached. Terminate the final node and return it. + + if end_token: + raise KconfigError("Unexpected end of file " + self._filename) + + prev.next = None + return prev + + def _parse_cond(self): + # Parses an optional 'if <expr>' construct and returns the parsed + # <expr>, or self.y if the next token is not _T_IF + + expr = self._parse_expr(True) if self._check_token(_T_IF) else self.y + if self._peek_token() is not None: + self._parse_error("extra tokens at end of line") + return expr + + def _parse_properties(self, node): + # Parses and adds properties to the MenuNode 'node' (type, 'prompt', + # 'default's, etc.) Properties are later copied up to symbols and + # choices in a separate pass after parsing, in _add_props_to_sc(). + # + # An older version of this code added properties directly to symbols + # and choices instead of to their menu nodes (and handled dependency + # propagation simultaneously), but that loses information on where a + # property is added when a symbol or choice is defined in multiple + # locations. Some Kconfig configuration systems rely heavily on such + # symbols, and better docs can be generated by keeping track of where + # properties are added. + # + # node: + # The menu node we're parsing properties on + + # Dependencies from 'depends on'. Will get propagated to the properties + # below. + node.dep = self.y + + while self._next_line(): + t0 = self._next_token() + if t0 is None: + continue + + if t0 in _TYPE_TOKENS: + self._set_type(node, _TOKEN_TO_TYPE[t0]) + if self._peek_token() is not None: + self._parse_prompt(node) + + elif t0 == _T_DEPENDS: + if not self._check_token(_T_ON): + self._parse_error('expected "on" after "depends"') + + node.dep = self._make_and(node.dep, self._parse_expr(True)) + + elif t0 == _T_HELP: + self._parse_help(node) + + elif t0 == _T_SELECT: + if not isinstance(node.item, Symbol): + self._parse_error("only symbols can select") + + node.selects.append((self._expect_nonconst_sym(), + self._parse_cond())) + + elif t0 == _T_IMPLY: + if not isinstance(node.item, Symbol): + self._parse_error("only symbols can imply") + + node.implies.append((self._expect_nonconst_sym(), + self._parse_cond())) + + elif t0 == _T_DEFAULT: + node.defaults.append((self._parse_expr(False), + self._parse_cond())) + + elif t0 in (_T_DEF_BOOL, _T_DEF_TRISTATE, _T_DEF_INT, _T_DEF_HEX, + _T_DEF_STRING): + self._set_type(node, _TOKEN_TO_TYPE[t0]) + node.defaults.append((self._parse_expr(False), + self._parse_cond())) + + elif t0 == _T_PROMPT: + self._parse_prompt(node) + + elif t0 == _T_RANGE: + node.ranges.append((self._expect_sym(), + self._expect_sym(), + self._parse_cond())) + + elif t0 == _T_OPTION: + if self._check_token(_T_ENV): + if not self._check_token(_T_EQUAL): + self._parse_error('expected "=" after "env"') + + env_var = self._expect_str_and_eol() + node.item.env_var = env_var + + if env_var in os.environ: + node.defaults.append( + (self._lookup_const_sym(os.environ[env_var]), + self.y)) + else: + self._warn("{1} has 'option env=\"{0}\"', " + "but the environment variable {0} is not " + "set".format(node.item.name, env_var), + self._filename, self._linenr) + + if env_var != node.item.name: + self._warn("Kconfiglib expands environment variables " + "in strings directly, meaning you do not " + "need 'option env=...' \"bounce\" symbols. " + "For compatibility with the C tools, " + "rename {} to {} (so that the symbol name " + "matches the environment variable name)." + .format(node.item.name, env_var), + self._filename, self._linenr) + + elif self._check_token(_T_DEFCONFIG_LIST): + if not self.defconfig_list: + self.defconfig_list = node.item + else: + self._warn("'option defconfig_list' set on multiple " + "symbols ({0} and {1}). Only {0} will be " + "used.".format(self.defconfig_list.name, + node.item.name), + self._filename, self._linenr) + + elif self._check_token(_T_MODULES): + # To reduce warning spam, only warn if 'option modules' is + # set on some symbol that isn't MODULES, which should be + # safe. I haven't run into any projects that make use + # modules besides the kernel yet, and there it's likely to + # keep being called "MODULES". + if node.item is not self.modules: + self._warn("the 'modules' option is not supported. " + "Let me know if this is a problem for you, " + "as it wouldn't be that hard to implement. " + "Note that modules are supported -- " + "Kconfiglib just assumes the symbol name " + "MODULES, like older versions of the C " + "implementation did when 'option modules' " + "wasn't used.", + self._filename, self._linenr) + + elif self._check_token(_T_ALLNOCONFIG_Y): + if not isinstance(node.item, Symbol): + self._parse_error("the 'allnoconfig_y' option is only " + "valid for symbols") + + node.item.is_allnoconfig_y = True + + else: + self._parse_error("unrecognized option") + + elif t0 == _T_VISIBLE: + if not self._check_token(_T_IF): + self._parse_error('expected "if" after "visible"') + + node.visibility = \ + self._make_and(node.visibility, self._parse_expr(True)) + + elif t0 == _T_OPTIONAL: + if not isinstance(node.item, Choice): + self._parse_error('"optional" is only valid for choices') + + node.item.is_optional = True + + else: + # Reuse the tokens for the non-property line later + self._has_tokens = True + self._tokens_i = -1 + return + + def _set_type(self, node, new_type): + if node.item.orig_type not in (UNKNOWN, new_type): + self._warn("{} defined with multiple types, {} will be used" + .format(_name_and_loc(node.item), + TYPE_TO_STR[new_type])) + + node.item.orig_type = new_type + + def _parse_prompt(self, node): + # 'prompt' properties override each other within a single definition of + # a symbol, but additional prompts can be added by defining the symbol + # multiple times + if node.prompt: + self._warn(_name_and_loc(node.item) + + " defined with multiple prompts in single location") + + prompt = self._expect_str() + if prompt != prompt.strip(): + self._warn(_name_and_loc(node.item) + + " has leading or trailing whitespace in its prompt") + + # This avoid issues for e.g. reStructuredText documentation, where + # '*prompt *' is invalid + prompt = prompt.strip() + + node.prompt = (prompt, self._parse_cond()) + + def _parse_help(self, node): + # Find first non-blank (not all-space) line and get its indentation + + if node.help is not None: + self._warn(_name_and_loc(node.item) + + " defined with more than one help text -- only the " + "last one will be used") + + # Small optimization. This code is pretty hot. + readline = self._file.readline + + while 1: + line = readline() + self._linenr += 1 + if not line or not line.isspace(): + break + + if not line: + self._warn(_name_and_loc(node.item) + + " has 'help' but empty help text") + + node.help = "" + return + + indent = _indentation(line) + if indent == 0: + # If the first non-empty lines has zero indent, there is no help + # text + self._warn(_name_and_loc(node.item) + + " has 'help' but empty help text") + + node.help = "" + self._saved_line = line # "Unget" the line + return + + # The help text goes on till the first non-empty line with less indent + # than the first line + + help_lines = [] + # Small optimizations + add_help_line = help_lines.append + indentation = _indentation + + while line and (line.isspace() or indentation(line) >= indent): + # De-indent 'line' by 'indent' spaces and rstrip() it to remove any + # newlines (which gets rid of other trailing whitespace too, but + # that's fine). + # + # This prepares help text lines in a speedy way: The [indent:] + # might already remove trailing newlines for lines shorter than + # indent (e.g. empty lines). The rstrip() makes it consistent, + # meaning we can join the lines with "\n" later. + add_help_line(line.expandtabs()[indent:].rstrip()) + + line = readline() + + self._linenr += len(help_lines) + + node.help = "\n".join(help_lines).rstrip() + "\n" + self._saved_line = line # "Unget" the line + + def _parse_expr(self, transform_m): + # Parses an expression from the tokens in Kconfig._tokens using a + # simple top-down approach. See the module docstring for the expression + # format. + # + # transform_m: + # True if m should be rewritten to m && MODULES. See the + # Kconfig.eval_string() documentation. + + # Grammar: + # + # expr: and_expr ['||' expr] + # and_expr: factor ['&&' and_expr] + # factor: <symbol> ['='/'!='/'<'/... <symbol>] + # '!' factor + # '(' expr ')' + # + # It helps to think of the 'expr: and_expr' case as a single-operand OR + # (no ||), and of the 'and_expr: factor' case as a single-operand AND + # (no &&). Parsing code is always a bit tricky. + + # Mind dump: parse_factor() and two nested loops for OR and AND would + # work as well. The straightforward implementation there gives a + # (op, (op, (op, A, B), C), D) parse for A op B op C op D. Representing + # expressions as (op, [list of operands]) instead goes nicely with that + # version, but is wasteful for short expressions and complicates + # expression evaluation and other code that works on expressions (more + # complicated code likely offsets any performance gain from less + # recursion too). If we also try to optimize the list representation by + # merging lists when possible (e.g. when ANDing two AND expressions), + # we end up allocating a ton of lists instead of reusing expressions, + # which is bad. + + and_expr = self._parse_and_expr(transform_m) + + # Return 'and_expr' directly if we have a "single-operand" OR. + # Otherwise, parse the expression on the right and make an OR node. + # This turns A || B || C || D into (OR, A, (OR, B, (OR, C, D))). + return and_expr \ + if not self._check_token(_T_OR) else \ + (OR, and_expr, self._parse_expr(transform_m)) + + def _parse_and_expr(self, transform_m): + factor = self._parse_factor(transform_m) + + # Return 'factor' directly if we have a "single-operand" AND. + # Otherwise, parse the right operand and make an AND node. This turns + # A && B && C && D into (AND, A, (AND, B, (AND, C, D))). + return factor \ + if not self._check_token(_T_AND) else \ + (AND, factor, self._parse_and_expr(transform_m)) + + def _parse_factor(self, transform_m): + token = self._next_token() + + if isinstance(token, Symbol): + # Plain symbol or relation + + next_token = self._peek_token() + if next_token not in _RELATIONS: + # Plain symbol + + # For conditional expressions ('depends on <expr>', + # '... if <expr>', etc.), m is rewritten to m && MODULES. + if transform_m and token is self.m: + return (AND, self.m, self.modules) + + return token + + # Relation + # + # _T_EQUAL, _T_UNEQUAL, etc., deliberately have the same values as + # EQUAL, UNEQUAL, etc., so we can just use the token directly + return (self._next_token(), token, self._expect_sym()) + + if token == _T_NOT: + # token == _T_NOT == NOT + return (token, self._parse_factor(transform_m)) + + if token == _T_OPEN_PAREN: + expr_parse = self._parse_expr(transform_m) + if self._check_token(_T_CLOSE_PAREN): + return expr_parse + + self._parse_error("malformed expression") + + # + # Caching and invalidation + # + + def _build_dep(self): + # Populates the Symbol/Choice._dependents sets, which contain all other + # items (symbols and choices) that immediately depend on the item in + # the sense that changing the value of the item might affect the value + # of the dependent items. This is used for caching/invalidation. + # + # The calculated sets might be larger than necessary as we don't do any + # complex analysis of the expressions. + + # Only calculate _dependents for defined symbols. Constant and + # undefined symbols could theoretically be selected/implied, but it + # wouldn't change their value, so it's not a true dependency. + for sym in self._defined_syms_set: + # Symbols depend on the following: + + # The prompt conditions + for node in sym.nodes: + if node.prompt: + _make_depend_on(sym, node.prompt[1]) + + # The default values and their conditions + for value, cond in sym.defaults: + _make_depend_on(sym, value) + _make_depend_on(sym, cond) + + # The reverse and weak reverse dependencies + _make_depend_on(sym, sym.rev_dep) + _make_depend_on(sym, sym.weak_rev_dep) + + # The ranges along with their conditions + for low, high, cond in sym.ranges: + _make_depend_on(sym, low) + _make_depend_on(sym, high) + _make_depend_on(sym, cond) + + # The direct dependencies. This is usually redundant, as the direct + # dependencies get propagated to properties, but it's needed to get + # invalidation solid for 'imply', which only checks the direct + # dependencies (even if there are no properties to propagate it + # to). + _make_depend_on(sym, sym.direct_dep) + + # In addition to the above, choice symbols depend on the choice + # they're in, but that's handled automatically since the Choice is + # propagated to the conditions of the properties before + # _build_dep() runs. + + for choice in self.choices: + # Choices depend on the following: + + # The prompt conditions + for node in choice.nodes: + if node.prompt: + _make_depend_on(choice, node.prompt[1]) + + # The default symbol conditions + for _, cond in choice.defaults: + _make_depend_on(choice, cond) + + def _add_choice_deps(self): + # Choices also depend on the choice symbols themselves, because the + # y-mode selection of the choice might change if a choice symbol's + # visibility changes. + # + # We add these dependencies separately after dependency loop detection. + # The invalidation algorithm can handle the resulting + # <choice symbol> <-> <choice> dependency loops, but they make loop + # detection awkward. + + for choice in self.choices: + # The choice symbols themselves, because the y mode selection might + # change if a choice symbol's visibility changes + for sym in choice.syms: + sym._dependents.add(choice) + + def _invalidate_all(self): + # Undefined symbols never change value and don't need to be + # invalidated, so we can just iterate over defined symbols. + # Invalidating constant symbols would break things horribly. + for sym in self._defined_syms_set: + sym._invalidate() + + for choice in self.choices: + choice._invalidate() + + + # + # Post-parsing menu tree processing, including dependency propagation and + # implicit submenu creation + # + + def _finalize_tree(self, node, visible_if): + # Propagates properties and dependencies, creates implicit menus (see + # kconfig-language.txt), removes 'if' nodes, and finalizes choices. + # This pretty closely mirrors menu_finalize() from the C + # implementation, with some minor tweaks (MenuNode holds lists of + # properties instead of each property having a MenuNode pointer, for + # example). + # + # node: + # The current "parent" menu node, from which we propagate + # dependencies + # + # visible_if: + # Dependencies from 'visible if' on parent menus. These are added to + # the prompts of symbols and choices. + + if node.list: + # The menu node is a choice, menu, or if. Finalize each child in + # it. + + if node.item == MENU: + visible_if = self._make_and(visible_if, node.visibility) + + # Propagate the menu node's dependencies to each child menu node. + # + # The recursive _finalize_tree() calls assume that the current + # "level" in the tree has already had dependencies propagated. This + # makes e.g. implicit submenu creation easier, because it needs to + # look ahead. + self._propagate_deps(node, visible_if) + + # Finalize the children + cur = node.list + while cur: + self._finalize_tree(cur, visible_if) + cur = cur.next + + elif isinstance(node.item, Symbol): + # Add the node's non-node-specific properties (defaults, ranges, + # etc.) to the Symbol + self._add_props_to_sc(node) + + # See if we can create an implicit menu rooted at the Symbol and + # finalize each child menu node in that menu if so, like for the + # choice/menu/if case above + cur = node + while cur.next and _auto_menu_dep(node, cur.next): + # This also makes implicit submenu creation work recursively, + # with implicit menus inside implicit menus + self._finalize_tree(cur.next, visible_if) + cur = cur.next + cur.parent = node + + if cur is not node: + # Found symbols that should go in an implicit submenu. Tilt + # them up above us. + node.list = node.next + node.next = cur.next + cur.next = None + + + if node.list: + # We have a parent node with individually finalized child nodes. Do + # final steps to finalize this "level" in the menu tree. + _flatten(node.list) + _remove_ifs(node) + + # Empty choices (node.list None) are possible, so this needs to go + # outside + if isinstance(node.item, Choice): + # Add the node's non-node-specific properties to the choice + self._add_props_to_sc(node) + _finalize_choice(node) + + def _propagate_deps(self, node, visible_if): + # Propagates 'node's dependencies to its child menu nodes + + # If the parent node holds a Choice, we use the Choice itself as the + # parent dependency. This makes sense as the value (mode) of the choice + # limits the visibility of the contained choice symbols. The C + # implementation works the same way. + # + # Due to the similar interface, Choice works as a drop-in replacement + # for Symbol here. + basedep = node.item if isinstance(node.item, Choice) else node.dep + + cur = node.list + while cur: + cur.dep = dep = self._make_and(cur.dep, basedep) + + # Propagate dependencies to prompt + if cur.prompt: + cur.prompt = (cur.prompt[0], + self._make_and(cur.prompt[1], dep)) + + if isinstance(cur.item, (Symbol, Choice)): + sc = cur.item + + # Propagate 'visible if' dependencies to the prompt + if cur.prompt: + cur.prompt = (cur.prompt[0], + self._make_and(cur.prompt[1], visible_if)) + + # Propagate dependencies to defaults + if cur.defaults: + cur.defaults = [(default, self._make_and(cond, dep)) + for default, cond in cur.defaults] + + # Propagate dependencies to ranges + if cur.ranges: + cur.ranges = [(low, high, self._make_and(cond, dep)) + for low, high, cond in cur.ranges] + + # Propagate dependencies to selects + if cur.selects: + cur.selects = [(target, self._make_and(cond, dep)) + for target, cond in cur.selects] + + # Propagate dependencies to implies + if cur.implies: + cur.implies = [(target, self._make_and(cond, dep)) + for target, cond in cur.implies] + + + cur = cur.next + + def _add_props_to_sc(self, node): + # Copies properties from the menu node 'node' up to its contained + # symbol or choice. + # + # This can't be rolled into _propagate_deps(), because that function + # traverses the menu tree roughly breadth-first order, meaning + # properties on symbols and choices defined in multiple locations could + # end up in the wrong order. + + # Symbol or choice + sc = node.item + + # See the Symbol class docstring + sc.direct_dep = self._make_or(sc.direct_dep, node.dep) + + if node.defaults: + sc.defaults.extend(node.defaults) + + if node.ranges: + sc.ranges.extend(node.ranges) + + if node.selects: + sc.selects.extend(node.selects) + + # Modify the reverse dependencies of the selected symbol + for target, cond in node.selects: + target.rev_dep = self._make_or( + target.rev_dep, + self._make_and(sc, cond)) + + if node.implies: + sc.implies.extend(node.implies) + + # Modify the weak reverse dependencies of the implied + # symbol + for target, cond in node.implies: + target.weak_rev_dep = self._make_or( + target.weak_rev_dep, + self._make_and(sc, cond)) + + + # + # Misc. + # + + def _parse_error(self, msg): + if self._filename is None: + loc = "" + else: + loc = "{}:{}: ".format(self._filename, self._linenr) + + raise KconfigError( + "{}couldn't parse '{}': {}".format(loc, self._line.rstrip(), msg)) + + def _open(self, filename, mode): + # open() wrapper: + # + # - Enable universal newlines mode on Python 2 to ease + # interoperability between Linux and Windows. It's already the + # default on Python 3. + # + # The "U" flag would currently work for both Python 2 and 3, but it's + # deprecated on Python 3, so play it future-safe. + # + # A simpler solution would be to use io.open(), which defaults to + # universal newlines on both Python 2 and 3 (and is an alias for + # open() on Python 3), but it's appreciably slower on Python 2: + # + # Parsing x86 Kconfigs on Python 2 + # + # with open(..., "rU"): + # + # real 0m0.930s + # user 0m0.905s + # sys 0m0.025s + # + # with io.open(): + # + # real 0m1.069s + # user 0m1.040s + # sys 0m0.029s + # + # There's no appreciable performance difference between "r" and + # "rU" for parsing performance on Python 2. + # + # - For Python 3, force the encoding. Forcing the encoding on Python 2 + # turns strings into Unicode strings, which gets messy. Python 2 + # doesn't decode regular strings anyway. + return open(filename, "rU" if mode == "r" else mode) if _IS_PY2 else \ + open(filename, mode, encoding=self._encoding) + + def _check_undefined_syms(self): + # Prints warnings for all references to undefined symbols within the + # Kconfig files + + for sym in (self.syms.viewvalues() if _IS_PY2 else self.syms.values()): + # - sym.nodes empty means the symbol is undefined (has no + # definition locations) + # + # - Due to Kconfig internals, numbers show up as undefined Kconfig + # symbols, but shouldn't be flagged + # + # - The MODULES symbol always exists + if not sym.nodes and not _is_num(sym.name) and \ + sym.name != "MODULES": + + self._warn_undefined_sym(sym) + + def _warn_undefined_sym(self, sym): + # _check_undefined_syms() helper. Generates a warning that lists the + # locations where the undefined symbol 'sym' is referenced, including + # the referencing menu nodes in Kconfig format. + + referencing_nodes = [] + + def find_refs(node): + while node: + if sym in node.referenced: + referencing_nodes.append(node) + + if node.list: + find_refs(node.list) + + node = node.next + + find_refs(self.top_node) + + msg = "undefined symbol {}:".format(sym.name) + + for node in referencing_nodes: + msg += "\n\n- Referenced at {}:{}:\n\n{}" \ + .format(node.filename, node.linenr, node) + + self._warn(msg) + + def _warn(self, msg, filename=None, linenr=None): + # For printing general warnings + + if self._warnings_enabled: + msg = "warning: " + msg + if filename is not None: + msg = "{}:{}: {}".format(filename, linenr, msg) + + self.warnings.append(msg) + if self._warn_to_stderr: + sys.stderr.write(msg + "\n") + + def _warn_undef_assign(self, msg, filename=None, linenr=None): + # See the class documentation + + if self._warn_for_undef_assign: + self._warn(msg, filename, linenr) + + def _warn_undef_assign_load(self, name, val, filename, linenr): + # Special version for load_config() + + self._warn_undef_assign( + 'attempt to assign the value "{}" to the undefined symbol {}' + .format(val, name), filename, linenr) + + def _warn_redun_assign(self, msg, filename=None, linenr=None): + # See the class documentation + + if self._warn_for_redun_assign: + self._warn(msg, filename, linenr) + + def _srctree_hint(self): + # Hint printed when Kconfig files can't be found or .config files can't + # be opened + + return ". Perhaps the $srctree environment variable (set to '{}') " \ + "is set incorrectly. Note that the current value of $srctree " \ + "is saved when the Kconfig instance is created (for " \ + "consistency and to cleanly separate instances)." \ + .format(self.srctree if self.srctree else "unset or blank") + +class Symbol(object): + """ + Represents a configuration symbol: + + (menu)config FOO + ... + + The following attributes are available. They should be viewed as read-only, + and some are implemented through @property magic (but are still efficient + to access due to internal caching). + + Note: Prompts, help texts, and locations are stored in the Symbol's + MenuNode(s) rather than in the Symbol itself. Check the MenuNode class and + the Symbol.nodes attribute. This organization matches the C tools. + + name: + The name of the symbol, e.g. "FOO" for 'config FOO'. + + type: + The type of the symbol. One of BOOL, TRISTATE, STRING, INT, HEX, UNKNOWN. + UNKNOWN is for undefined symbols, (non-special) constant symbols, and + symbols defined without a type. + + When running without modules (MODULES having the value n), TRISTATE + symbols magically change type to BOOL. This also happens for symbols + within choices in "y" mode. This matches the C tools, and makes sense for + menuconfig-like functionality. + + orig_type: + The type as given in the Kconfig file, without any magic applied. Used + when printing the symbol. + + str_value: + The value of the symbol as a string. Gives the value for string/int/hex + symbols. For bool/tristate symbols, gives "n", "m", or "y". + + This is the symbol value that's used in relational expressions + (A = B, A != B, etc.) + + Gotcha: For int/hex symbols, the exact format of the value must often be + preserved (e.g., when writing a .config file), hence why you can't get it + directly as an int. Do int(int_sym.str_value) or + int(hex_sym.str_value, 16) to get the integer value. + + tri_value: + The tristate value of the symbol as an integer. One of 0, 1, 2, + representing n, m, y. Always 0 (n) for non-bool/tristate symbols. + + This is the symbol value that's used outside of relation expressions + (A, !A, A && B, A || B). + + assignable: + A tuple containing the tristate user values that can currently be + assigned to the symbol (that would be respected), ordered from lowest (0, + representing n) to highest (2, representing y). This corresponds to the + selections available in the menuconfig interface. The set of assignable + values is calculated from the symbol's visibility and selects/implies. + + Returns the empty set for non-bool/tristate symbols and for symbols with + visibility n. The other possible values are (0, 2), (0, 1, 2), (1, 2), + (1,), and (2,). A (1,) or (2,) result means the symbol is visible but + "locked" to m or y through a select, perhaps in combination with the + visibility. menuconfig represents this as -M- and -*-, respectively. + + For string/hex/int symbols, check if Symbol.visibility is non-0 (non-n) + instead to determine if the value can be changed. + + Some handy 'assignable' idioms: + + # Is 'sym' an assignable (visible) bool/tristate symbol? + if sym.assignable: + # What's the highest value it can be assigned? [-1] in Python + # gives the last element. + sym_high = sym.assignable[-1] + + # The lowest? + sym_low = sym.assignable[0] + + # Can the symbol be set to at least m? + if sym.assignable[-1] >= 1: + ... + + # Can the symbol be set to m? + if 1 in sym.assignable: + ... + + visibility: + The visibility of the symbol. One of 0, 1, 2, representing n, m, y. See + the module documentation for an overview of symbol values and visibility. + + user_value: + The user value of the symbol. None if no user value has been assigned + (via Kconfig.load_config() or Symbol.set_value()). + + Holds 0, 1, or 2 for bool/tristate symbols, and a string for the other + symbol types. + + WARNING: Do not assign directly to this. It will break things. Use + Symbol.set_value(). + + config_string: + The .config assignment string that would get written out for the symbol + by Kconfig.write_config(). Returns the empty string if no .config + assignment would get written out. In general, visible symbols, symbols + with (active) defaults, and selected symbols get written out. + + nodes: + A list of MenuNodes for this symbol. Will contain a single MenuNode for + most symbols. Undefined and constant symbols have an empty nodes list. + Symbols defined in multiple locations get one node for each location. + + choice: + Holds the parent Choice for choice symbols, and None for non-choice + symbols. Doubles as a flag for whether a symbol is a choice symbol. + + defaults: + List of (default, cond) tuples for the symbol's 'default' properties. For + example, 'default A && B if C || D' is represented as + ((AND, A, B), (OR, C, D)). If no condition was given, 'cond' is + self.kconfig.y. + + Note that 'depends on' and parent dependencies are propagated to + 'default' conditions. + + selects: + List of (symbol, cond) tuples for the symbol's 'select' properties. For + example, 'select A if B && C' is represented as (A, (AND, B, C)). If no + condition was given, 'cond' is self.kconfig.y. + + Note that 'depends on' and parent dependencies are propagated to 'select' + conditions. + + implies: + Like 'selects', for imply. + + ranges: + List of (low, high, cond) tuples for the symbol's 'range' properties. For + example, 'range 1 2 if A' is represented as (1, 2, A). If there is no + condition, 'cond' is self.config.y. + + Note that 'depends on' and parent dependencies are propagated to 'range' + conditions. + + Gotcha: 1 and 2 above will be represented as (undefined) Symbols rather + than plain integers. Undefined symbols get their name as their string + value, so this works out. The C tools work the same way. + + rev_dep: + Reverse dependency expression from other symbols selecting this symbol. + Multiple selections get ORed together. A condition on a select is ANDed + with the selecting symbol. + + For example, if A has 'select FOO' and B has 'select FOO if C', then + FOO's rev_dep will be (OR, A, (AND, B, C)). + + weak_rev_dep: + Like rev_dep, for imply. + + direct_dep: + The 'depends on' dependencies. If a symbol is defined in multiple + locations, the dependencies at each location are ORed together. + + Internally, this is used to implement 'imply', which only applies if the + implied symbol has expr_value(self.direct_dep) != 0. 'depends on' and + parent dependencies are automatically propagated to the conditions of + properties, so normally it's redundant to check the direct dependencies. + + referenced: + A set() with all symbols and choices referenced in the properties and + property conditions of the symbol. + + Also includes dependencies inherited from surrounding menus and if's. + Choices appear in the dependencies of choice symbols. + + env_var: + If the Symbol has an 'option env="FOO"' option, this contains the name + ("FOO") of the environment variable. None for symbols without no + 'option env'. + + 'option env="FOO"' acts like a 'default' property whose value is the + value of $FOO. + + Symbols with 'option env' are never written out to .config files, even if + they are visible. env_var corresponds to a flag called SYMBOL_AUTO in the + C implementation. + + is_allnoconfig_y: + True if the symbol has 'option allnoconfig_y' set on it. This has no + effect internally (except when printing symbols), but can be checked by + scripts. + + is_constant: + True if the symbol is a constant (quoted) symbol. + + kconfig: + The Kconfig instance this symbol is from. + """ + __slots__ = ( + "_cached_assignable", + "_cached_str_val", + "_cached_tri_val", + "_cached_vis", + "_checked", + "_dependents", + "_old_val", + "_was_set", + "_write_to_conf", + "_written", + "choice", + "defaults", + "direct_dep", + "env_var", + "implies", + "is_allnoconfig_y", + "is_constant", + "kconfig", + "name", + "nodes", + "orig_type", + "ranges", + "rev_dep", + "selects", + "user_value", + "weak_rev_dep", + ) + + # + # Public interface + # + + @property + def type(self): + """ + See the class documentation. + """ + if self.orig_type == TRISTATE and \ + ((self.choice and self.choice.tri_value == 2) or + not self.kconfig.modules.tri_value): + return BOOL + + return self.orig_type + + @property + def str_value(self): + """ + See the class documentation. + """ + if self._cached_str_val is not None: + return self._cached_str_val + + if self.orig_type in (BOOL, TRISTATE): + # Also calculates the visibility, so invalidation safe + self._cached_str_val = TRI_TO_STR[self.tri_value] + return self._cached_str_val + + # As a quirk of Kconfig, undefined symbols get their name as their + # string value. This is why things like "FOO = bar" work for seeing if + # FOO has the value "bar". + if self.orig_type == UNKNOWN: + self._cached_str_val = self.name + return self.name + + val = "" + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + + self._write_to_conf = (vis != 0) + + if self.orig_type in (INT, HEX): + # The C implementation checks the user value against the range in a + # separate code path (post-processing after loading a .config). + # Checking all values here instead makes more sense for us. It + # requires that we check for a range first. + + base = _TYPE_TO_BASE[self.orig_type] + + # Check if a range is in effect + for low_expr, high_expr, cond in self.ranges: + if expr_value(cond): + has_active_range = True + + # The zeros are from the C implementation running strtoll() + # on empty strings + low = int(low_expr.str_value, base) if \ + _is_base_n(low_expr.str_value, base) else 0 + high = int(high_expr.str_value, base) if \ + _is_base_n(high_expr.str_value, base) else 0 + + break + else: + has_active_range = False + + # Defaults are used if the symbol is invisible, lacks a user value, + # or has an out-of-range user value. + use_defaults = True + + if vis and self.user_value: + user_val = int(self.user_value, base) + if has_active_range and not low <= user_val <= high: + num2str = str if base == 10 else hex + self.kconfig._warn( + "user value {} on the {} symbol {} ignored due to " + "being outside the active range ([{}, {}]) -- falling " + "back on defaults" + .format(num2str(user_val), TYPE_TO_STR[self.orig_type], + _name_and_loc(self), + num2str(low), num2str(high))) + else: + # If the user value is well-formed and satisfies range + # contraints, it is stored in exactly the same form as + # specified in the assignment (with or without "0x", etc.) + val = self.user_value + use_defaults = False + + if use_defaults: + # No user value or invalid user value. Look at defaults. + + # Used to implement the warning below + has_default = False + + for val_sym, cond in self.defaults: + if expr_value(cond): + has_default = self._write_to_conf = True + + val = val_sym.str_value + + if _is_base_n(val, base): + val_num = int(val, base) + else: + val_num = 0 # strtoll() on empty string + + break + else: + val_num = 0 # strtoll() on empty string + + # This clamping procedure runs even if there's no default + if has_active_range: + clamp = None + if val_num < low: + clamp = low + elif val_num > high: + clamp = high + + if clamp is not None: + # The value is rewritten to a standard form if it is + # clamped + val = str(clamp) \ + if self.orig_type == INT else \ + hex(clamp) + + if has_default: + num2str = str if base == 10 else hex + self.kconfig._warn( + "default value {} on {} clamped to {} due to " + "being outside the active range ([{}, {}])" + .format(val_num, _name_and_loc(self), + num2str(clamp), num2str(low), + num2str(high))) + + elif self.orig_type == STRING: + if vis and self.user_value is not None: + # If the symbol is visible and has a user value, use that + val = self.user_value + else: + # Otherwise, look at defaults + for val_sym, cond in self.defaults: + if expr_value(cond): + val = val_sym.str_value + self._write_to_conf = True + break + + # env_var corresponds to SYMBOL_AUTO in the C implementation, and is + # also set on the defconfig_list symbol there. Test for the + # defconfig_list symbol explicitly instead here, to avoid a nonsensical + # env_var setting and the defconfig_list symbol being printed + # incorrectly. This code is pretty cold anyway. + if self.env_var is not None or self is self.kconfig.defconfig_list: + self._write_to_conf = False + + self._cached_str_val = val + return val + + @property + def tri_value(self): + """ + See the class documentation. + """ + if self._cached_tri_val is not None: + return self._cached_tri_val + + if self.orig_type not in (BOOL, TRISTATE): + if self.orig_type != UNKNOWN: + # Would take some work to give the location here + self.kconfig._warn( + "The {} symbol {} is being evaluated in a logical context " + "somewhere. It will always evaluate to n." + .format(TYPE_TO_STR[self.orig_type], _name_and_loc(self))) + + self._cached_tri_val = 0 + return 0 + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + self._write_to_conf = (vis != 0) + + val = 0 + + if not self.choice: + # Non-choice symbol + + if vis and self.user_value is not None: + # If the symbol is visible and has a user value, use that + val = min(self.user_value, vis) + + else: + # Otherwise, look at defaults and weak reverse dependencies + # (implies) + + for default, cond in self.defaults: + cond_val = expr_value(cond) + if cond_val: + val = min(expr_value(default), cond_val) + if val: + self._write_to_conf = True + break + + # Weak reverse dependencies are only considered if our + # direct dependencies are met + weak_rev_dep_val = expr_value(self.weak_rev_dep) + if weak_rev_dep_val and expr_value(self.direct_dep): + val = max(weak_rev_dep_val, val) + self._write_to_conf = True + + # Reverse (select-related) dependencies take precedence + rev_dep_val = expr_value(self.rev_dep) + if rev_dep_val: + if expr_value(self.direct_dep) < rev_dep_val: + self._warn_select_unsatisfied_deps() + + val = max(rev_dep_val, val) + self._write_to_conf = True + + # m is promoted to y for (1) bool symbols and (2) symbols with a + # weak_rev_dep (from imply) of y + if val == 1 and \ + (self.type == BOOL or expr_value(self.weak_rev_dep) == 2): + val = 2 + + elif vis == 2: + # Visible choice symbol in y-mode choice. The choice mode limits + # the visibility of choice symbols, so it's sufficient to just + # check the visibility of the choice symbols themselves. + val = 2 if self.choice.selection is self else 0 + + elif vis and self.user_value: + # Visible choice symbol in m-mode choice, with set non-0 user value + val = 1 + + self._cached_tri_val = val + return val + + @property + def assignable(self): + """ + See the class documentation. + """ + if self._cached_assignable is None: + self._cached_assignable = self._assignable() + + return self._cached_assignable + + @property + def visibility(self): + """ + See the class documentation. + """ + if self._cached_vis is None: + self._cached_vis = _visibility(self) + + return self._cached_vis + + @property + def config_string(self): + """ + See the class documentation. + """ + # Note: _write_to_conf is determined when the value is calculated. This + # is a hidden function call due to property magic. + val = self.str_value + if not self._write_to_conf: + return "" + + if self.orig_type in (BOOL, TRISTATE): + return "{}{}={}\n" \ + .format(self.kconfig.config_prefix, self.name, val) \ + if val != "n" else \ + "# {}{} is not set\n" \ + .format(self.kconfig.config_prefix, self.name) + + if self.orig_type in (INT, HEX): + return "{}{}={}\n" \ + .format(self.kconfig.config_prefix, self.name, val) + + if self.orig_type == STRING: + return '{}{}="{}"\n' \ + .format(self.kconfig.config_prefix, self.name, escape(val)) + + _internal_error("Internal error while creating .config: unknown " + 'type "{}".'.format(self.orig_type)) + + def set_value(self, value): + """ + Sets the user value of the symbol. + + Equal in effect to assigning the value to the symbol within a .config + file. For bool and tristate symbols, use the 'assignable' attribute to + check which values can currently be assigned. Setting values outside + 'assignable' will cause Symbol.user_value to differ from + Symbol.str/tri_value (be truncated down or up). + + Setting a choice symbol to 2 (y) sets Choice.user_selection to the + choice symbol in addition to setting Symbol.user_value. + Choice.user_selection is considered when the choice is in y mode (the + "normal" mode). + + Other symbols that depend (possibly indirectly) on this symbol are + automatically recalculated to reflect the assigned value. + + value: + The user value to give to the symbol. For bool and tristate symbols, + n/m/y can be specified either as 0/1/2 (the usual format for tristate + values in Kconfiglib) or as one of the strings "n"/"m"/"y". For other + symbol types, pass a string. + + Values that are invalid for the type (such as "foo" or 1 (m) for a + BOOL or "0x123" for an INT) are ignored and won't be stored in + Symbol.user_value. Kconfiglib will print a warning by default for + invalid assignments, and set_value() will return False. + + Returns True if the value is valid for the type of the symbol, and + False otherwise. This only looks at the form of the value. For BOOL and + TRISTATE symbols, check the Symbol.assignable attribute to see what + values are currently in range and would actually be reflected in the + value of the symbol. For other symbol types, check whether the + visibility is non-n. + """ + # If the new user value matches the old, nothing changes, and we can + # save some work. + # + # This optimization is skipped for choice symbols: Setting a choice + # symbol's user value to y might change the state of the choice, so it + # wouldn't be safe (symbol user values always match the values set in a + # .config file or via set_value(), and are never implicitly updated). + if value == self.user_value and not self.choice: + self._was_set = True + return True + + # Check if the value is valid for our type + if not (self.orig_type == BOOL and value in (0, 2, "n", "y") or + self.orig_type == TRISTATE and value in (0, 1, 2, "n", "m", "y") or + (isinstance(value, str) and + (self.orig_type == STRING or + self.orig_type == INT and _is_base_n(value, 10) or + self.orig_type == HEX and _is_base_n(value, 16) + and int(value, 16) >= 0))): + + # Display tristate values as n, m, y in the warning + self.kconfig._warn( + "the value {} is invalid for {}, which has type {} -- " + "assignment ignored" + .format(TRI_TO_STR[value] if value in (0, 1, 2) else + "'{}'".format(value), + _name_and_loc(self), + TYPE_TO_STR[self.orig_type])) + + return False + + if self.env_var is not None: + self.kconfig._warn("ignored attempt to assign user value to " + "{}, which is set from the environment" + .format(_name_and_loc(self))) + return False + + if self.orig_type in (BOOL, TRISTATE) and value in ("n", "m", "y"): + value = STR_TO_TRI[value] + + self.user_value = value + self._was_set = True + + if self.choice and value == 2: + # Setting a choice symbol to y makes it the user selection of the + # choice. Like for symbol user values, the user selection is not + # guaranteed to match the actual selection of the choice, as + # dependencies come into play. + self.choice.user_selection = self + self.choice._was_set = True + self.choice._rec_invalidate() + else: + self._rec_invalidate_if_has_prompt() + + return True + + def unset_value(self): + """ + Resets the user value of the symbol, as if the symbol had never gotten + a user value via Kconfig.load_config() or Symbol.set_value(). + """ + if self.user_value is not None: + self.user_value = None + self._rec_invalidate_if_has_prompt() + + @property + def referenced(self): + """ + See the class documentation. + """ + res = set() + for node in self.nodes: + res |= node.referenced + + return res + + def __repr__(self): + """ + Returns a string with information about the symbol (including its name, + value, visibility, and location(s)) when it is evaluated on e.g. the + interactive Python prompt. + """ + fields = [] + + fields.append("symbol " + self.name) + fields.append(TYPE_TO_STR[self.type]) + + for node in self.nodes: + if node.prompt: + fields.append('"{}"'.format(node.prompt[0])) + + # Only add quotes for non-bool/tristate symbols + fields.append("value " + + (self.str_value + if self.orig_type in (BOOL, TRISTATE) else + '"{}"'.format(self.str_value))) + + if not self.is_constant: + # These aren't helpful to show for constant symbols + + if self.user_value is not None: + # Only add quotes for non-bool/tristate symbols + fields.append("user value " + + (TRI_TO_STR[self.user_value] + if self.orig_type in (BOOL, TRISTATE) else + '"{}"'.format(self.user_value))) + + fields.append("visibility " + TRI_TO_STR[self.visibility]) + + if self.choice: + fields.append("choice symbol") + + if self.is_allnoconfig_y: + fields.append("allnoconfig_y") + + if self is self.kconfig.defconfig_list: + fields.append("is the defconfig_list symbol") + + if self.env_var is not None: + fields.append("from environment variable " + self.env_var) + + if self is self.kconfig.modules: + fields.append("is the modules symbol") + + fields.append("direct deps " + + TRI_TO_STR[expr_value(self.direct_dep)]) + + if self.nodes: + for node in self.nodes: + fields.append("{}:{}".format(node.filename, node.linenr)) + else: + if self.is_constant: + fields.append("constant") + else: + fields.append("undefined") + + return "<{}>".format(", ".join(fields)) + + def __str__(self): + """ + Returns a string representation of the symbol when it is printed, + matching the Kconfig format, with parent dependencies propagated. + + The string is constructed by joining the strings returned by + MenuNode.__str__() for each of the symbol's menu nodes, so symbols + defined in multiple locations will return a string with all + definitions. + + An empty string is returned for undefined and constant symbols. + """ + return "\n".join(str(node) for node in self.nodes) + + # + # Private methods + # + + def __init__(self): + """ + Symbol constructor -- not intended to be called directly by Kconfiglib + clients. + """ + # These attributes are always set on the instance from outside and + # don't need defaults: + # _written + # kconfig + # direct_dep + # is_constant + # name + # rev_dep + # weak_rev_dep + + self.orig_type = UNKNOWN + self.defaults = [] + self.selects = [] + self.implies = [] + self.ranges = [] + + self.nodes = [] + + self.user_value = \ + self.choice = \ + self.env_var = \ + self._cached_str_val = self._cached_tri_val = self._cached_vis = \ + self._cached_assignable = None + + # _write_to_conf is calculated along with the value. If True, the + # Symbol gets a .config entry. + + self.is_allnoconfig_y = \ + self._was_set = \ + self._write_to_conf = False + + # See Kconfig._build_dep() + self._dependents = set() + + # Used during dependency loop detection + self._checked = 0 + + def _assignable(self): + # Worker function for the 'assignable' attribute + + if self.orig_type not in (BOOL, TRISTATE): + return () + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + + if not vis: + return () + + rev_dep_val = expr_value(self.rev_dep) + + if vis == 2: + if self.choice: + return (2,) + + if not rev_dep_val: + if self.type == BOOL or expr_value(self.weak_rev_dep) == 2: + return (0, 2) + return (0, 1, 2) + + if rev_dep_val == 2: + return (2,) + + # rev_dep_val == 1 + + if self.type == BOOL or expr_value(self.weak_rev_dep) == 2: + return (2,) + return (1, 2) + + # vis == 1 + + # Must be a tristate here, because bool m visibility gets promoted to y + + if not rev_dep_val: + return (0, 1) if expr_value(self.weak_rev_dep) != 2 else (0, 2) + + if rev_dep_val == 2: + return (2,) + + # vis == rev_dep_val == 1 + + return (1,) + + def _invalidate(self): + # Marks the symbol as needing to be recalculated + + self._cached_str_val = self._cached_tri_val = self._cached_vis = \ + self._cached_assignable = None + + def _rec_invalidate(self): + # Invalidates the symbol and all items that (possibly) depend on it + + if self is self.kconfig.modules: + # Invalidating MODULES has wide-ranging effects + self.kconfig._invalidate_all() + else: + self._invalidate() + + for item in self._dependents: + # _cached_vis doubles as a flag that tells us whether 'item' + # has cached values, because it's calculated as a side effect + # of calculating all other (non-constant) cached values. + # + # If item._cached_vis is None, it means there can't be cached + # values on other items that depend on 'item', because if there + # were, some value on 'item' would have been calculated and + # item._cached_vis set as a side effect. It's therefore safe to + # stop the invalidation at symbols with _cached_vis None. + # + # This approach massively speeds up scripts that set a lot of + # values, vs simply invalidating all possibly dependent symbols + # (even when you already have a list of all the dependent + # symbols, because some symbols get huge dependency trees). + # + # This gracefully handles dependency loops too, which is nice + # for choices, where the choice depends on the choice symbols + # and vice versa. + if item._cached_vis is not None: + item._rec_invalidate() + + def _rec_invalidate_if_has_prompt(self): + # Invalidates the symbol and its dependent symbols, but only if the + # symbol has a prompt. User values never have an effect on promptless + # symbols, so we skip invalidation for them as an optimization. + # + # This also prevents constant (quoted) symbols from being invalidated + # if set_value() is called on them, which would cause them to lose + # their value and break things. + # + # Prints a warning if the symbol has no prompt. In some contexts (e.g. + # when loading a .config files) assignments to promptless symbols are + # normal and expected, so the warning can be disabled. + + for node in self.nodes: + if node.prompt: + self._rec_invalidate() + return + + if self.kconfig._warn_for_no_prompt: + self.kconfig._warn(_name_and_loc(self) + " has no prompt, meaning " + "user values have no effect on it") + + def _str_default(self): + # write_min_config() helper function. Returns the value the symbol + # would get from defaults if it didn't have a user value. Uses exactly + # the same algorithm as the C implementation (though a bit cleaned up), + # for compatibility. + + if self.orig_type in (BOOL, TRISTATE): + val = 0 + + # Defaults, selects, and implies do not affect choice symbols + if not self.choice: + for default, cond in self.defaults: + cond_val = expr_value(cond) + if cond_val: + val = min(expr_value(default), cond_val) + break + + val = max(expr_value(self.rev_dep), + expr_value(self.weak_rev_dep), + val) + + # Transpose mod to yes if type is bool (possibly due to modules + # being disabled) + if val == 1 and self.type == BOOL: + val = 2 + + return TRI_TO_STR[val] + + if self.orig_type in (STRING, INT, HEX): + for default, cond in self.defaults: + if expr_value(cond): + return default.str_value + + return "" + + def _warn_select_unsatisfied_deps(self): + # Helper for printing an informative warning when a symbol with + # unsatisfied direct dependencies (dependencies from 'depends on', ifs, + # and menus) is selected by some other symbol. Also warn if a symbol + # whose direct dependencies evaluate to m is selected to y. + + dir_dep_val = expr_value(self.direct_dep) + + msg = "{} has direct dependencies {} with value {}, but is " \ + "currently being {}-selected by the following symbols:" \ + .format(_name_and_loc(self), expr_str(self.direct_dep), + TRI_TO_STR[dir_dep_val], + TRI_TO_STR[expr_value(self.rev_dep)]) + + # The reverse dependencies from each select are ORed together + for select in split_expr(self.rev_dep, OR): + select_val = expr_value(select) + if select_val <= dir_dep_val: + # Only include selects that exceed the direct dependencies + continue + + # - 'select A if B' turns into A && B + # - 'select A' just turns into A + # + # In both cases, we can split on AND and pick the first operand + selecting_sym = split_expr(select, AND)[0] + + msg += "\n - {}, with value {}, direct dependencies {} " \ + "(value: {})" \ + .format(_name_and_loc(selecting_sym), + selecting_sym.str_value, + expr_str(selecting_sym.direct_dep), + TRI_TO_STR[expr_value(selecting_sym.direct_dep)]) + + if isinstance(select, tuple): + msg += ", and select condition {} (value: {})" \ + .format(expr_str(select[2]), + TRI_TO_STR[expr_value(select[2])]) + + self.kconfig._warn(msg) + +class Choice(object): + """ + Represents a choice statement: + + choice + ... + endchoice + + The following attributes are available on Choice instances. They should be + treated as read-only, and some are implemented through @property magic (but + are still efficient to access due to internal caching). + + Note: Prompts, help texts, and locations are stored in the Choice's + MenuNode(s) rather than in the Choice itself. Check the MenuNode class and + the Choice.nodes attribute. This organization matches the C tools. + + name: + The name of the choice, e.g. "FOO" for 'choice FOO', or None if the + Choice has no name. I can't remember ever seeing named choices in + practice, but the C tools support them too. + + type: + The type of the choice. One of BOOL, TRISTATE, UNKNOWN. UNKNOWN is for + choices defined without a type where none of the contained symbols have a + type either (otherwise the choice inherits the type of the first symbol + defined with a type). + + When running without modules (CONFIG_MODULES=n), TRISTATE choices + magically change type to BOOL. This matches the C tools, and makes sense + for menuconfig-like functionality. + + orig_type: + The type as given in the Kconfig file, without any magic applied. Used + when printing the choice. + + tri_value: + The tristate value (mode) of the choice. A choice can be in one of three + modes: + + 0 (n) - The choice is disabled and no symbols can be selected. For + visible choices, this mode is only possible for choices with + the 'optional' flag set (see kconfig-language.txt). + + 1 (m) - Any number of choice symbols can be set to m, the rest will + be n. + + 2 (y) - One symbol will be y, the rest n. + + Only tristate choices can be in m mode. The visibility of the choice is + an upper bound on the mode, and the mode in turn is an upper bound on the + visibility of the choice symbols. + + To change the mode, use Choice.set_value(). + + Implementation note: + The C tools internally represent choices as a type of symbol, with + special-casing in many code paths. This is why there is a lot of + similarity to Symbol. The value (mode) of a choice is really just a + normal symbol value, and an implicit reverse dependency forces its + lower bound to m for visible non-optional choices (the reverse + dependency is 'm && <visibility>'). + + Symbols within choices get the choice propagated as a dependency to + their properties. This turns the mode of the choice into an upper bound + on e.g. the visibility of choice symbols, and explains the gotcha + related to printing choice symbols mentioned in the module docstring. + + Kconfiglib uses a separate Choice class only because it makes the code + and interface less confusing (especially in a user-facing interface). + Corresponding attributes have the same name in the Symbol and Choice + classes, for consistency and compatibility. + + assignable: + See the symbol class documentation. Gives the assignable values (modes). + + visibility: + See the Symbol class documentation. Acts on the value (mode). + + selection: + The Symbol instance of the currently selected symbol. None if the Choice + is not in y mode or has no selected symbol (due to unsatisfied + dependencies on choice symbols). + + WARNING: Do not assign directly to this. It will break things. Call + sym.set_value(2) on the choice symbol you want to select instead. + + user_value: + The value (mode) selected by the user through Choice.set_value(). Either + 0, 1, or 2, or None if the user hasn't selected a mode. See + Symbol.user_value. + + WARNING: Do not assign directly to this. It will break things. Use + Choice.set_value() instead. + + user_selection: + The symbol selected by the user (by setting it to y). Ignored if the + choice is not in y mode, but still remembered so that the choice "snaps + back" to the user selection if the mode is changed back to y. This might + differ from 'selection' due to unsatisfied dependencies. + + WARNING: Do not assign directly to this. It will break things. Call + sym.set_value(2) on the choice symbol to be selected instead. + + syms: + List of symbols contained in the choice. + + Gotcha: If a symbol depends on the previous symbol within a choice so + that an implicit menu is created, it won't be a choice symbol, and won't + be included in 'syms'. There are real-world examples of this, and it was + a PITA to support in older versions of Kconfiglib that didn't implement + the menu structure. + + nodes: + A list of MenuNodes for this choice. In practice, the list will probably + always contain a single MenuNode, but it is possible to give a choice a + name and define it in multiple locations (I've never even seen a named + choice though). + + defaults: + List of (symbol, cond) tuples for the choice's 'defaults' properties. For + example, 'default A if B && C' is represented as (A, (AND, B, C)). If + there is no condition, 'cond' is self.config.y. + + Note that 'depends on' and parent dependencies are propagated to + 'default' conditions. + + direct_dep: + See Symbol.direct_dep. + + referenced: + A set() with all symbols referenced in the properties and property + conditions of the choice. + + Also includes dependencies inherited from surrounding menus and if's. + + is_optional: + True if the choice has the 'optional' flag set on it and can be in + n mode. + + kconfig: + The Kconfig instance this choice is from. + """ + __slots__ = ( + "_cached_assignable", + "_cached_selection", + "_cached_vis", + "_checked", + "_dependents", + "_was_set", + "defaults", + "direct_dep", + "is_constant", + "is_optional", + "kconfig", + "name", + "nodes", + "orig_type", + "syms", + "user_selection", + "user_value", + ) + + # + # Public interface + # + + @property + def type(self): + """ + Returns the type of the choice. See Symbol.type. + """ + if self.orig_type == TRISTATE and not self.kconfig.modules.tri_value: + return BOOL + + return self.orig_type + + @property + def str_value(self): + """ + See the class documentation. + """ + return TRI_TO_STR[self.tri_value] + + @property + def tri_value(self): + """ + See the class documentation. + """ + # This emulates a reverse dependency of 'm && visibility' for + # non-optional choices, which is how the C implementation does it + + val = 0 if self.is_optional else 1 + + if self.user_value is not None: + val = max(val, self.user_value) + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + val = min(val, self.visibility) + + # Promote m to y for boolean choices + return 2 if val == 1 and self.type == BOOL else val + + @property + def assignable(self): + """ + See the class documentation. + """ + if self._cached_assignable is None: + self._cached_assignable = self._assignable() + + return self._cached_assignable + + @property + def visibility(self): + """ + See the class documentation. + """ + if self._cached_vis is None: + self._cached_vis = _visibility(self) + + return self._cached_vis + + @property + def selection(self): + """ + See the class documentation. + """ + if self._cached_selection is _NO_CACHED_SELECTION: + self._cached_selection = self._selection() + + return self._cached_selection + + def set_value(self, value): + """ + Sets the user value (mode) of the choice. Like for Symbol.set_value(), + the visibility might truncate the value. Choices without the 'optional' + attribute (is_optional) can never be in n mode, but 0/"n" is still + accepted since it's not a malformed value (though it will have no + effect). + + Returns True if the value is valid for the type of the choice, and + False otherwise. This only looks at the form of the value. Check the + Choice.assignable attribute to see what values are currently in range + and would actually be reflected in the mode of the choice. + """ + if value == self.user_value: + # We know the value must be valid if it was successfully set + # previously + self._was_set = True + return True + + if not ((self.orig_type == BOOL and value in (0, 2, "n", "y") ) or + (self.orig_type == TRISTATE and value in (0, 1, 2, "n", "m", "y"))): + + # Display tristate values as n, m, y in the warning + self.kconfig._warn( + "the value {} is invalid for {}, which has type {} -- " + "assignment ignored" + .format(TRI_TO_STR[value] if value in (0, 1, 2) else + "'{}'".format(value), + _name_and_loc(self), + TYPE_TO_STR[self.orig_type])) + + return False + + if value in ("n", "m", "y"): + value = STR_TO_TRI[value] + + self.user_value = value + self._was_set = True + self._rec_invalidate() + + return True + + def unset_value(self): + """ + Resets the user value (mode) and user selection of the Choice, as if + the user had never touched the mode or any of the choice symbols. + """ + if self.user_value is not None or self.user_selection: + self.user_value = self.user_selection = None + self._rec_invalidate() + + @property + def referenced(self): + """ + See the class documentation. + """ + res = set() + for node in self.nodes: + res |= node.referenced + + return res + + def __repr__(self): + """ + Returns a string with information about the choice when it is evaluated + on e.g. the interactive Python prompt. + """ + fields = [] + + fields.append("choice" if self.name is None else \ + "choice " + self.name) + fields.append(TYPE_TO_STR[self.type]) + + for node in self.nodes: + if node.prompt: + fields.append('"{}"'.format(node.prompt[0])) + + fields.append("mode " + self.str_value) + + if self.user_value is not None: + fields.append('user mode {}'.format(TRI_TO_STR[self.user_value])) + + if self.selection: + fields.append("{} selected".format(self.selection.name)) + + if self.user_selection: + user_sel_str = "{} selected by user" \ + .format(self.user_selection.name) + + if self.selection is not self.user_selection: + user_sel_str += " (overridden)" + + fields.append(user_sel_str) + + fields.append("visibility " + TRI_TO_STR[self.visibility]) + + if self.is_optional: + fields.append("optional") + + for node in self.nodes: + fields.append("{}:{}".format(node.filename, node.linenr)) + + return "<{}>".format(", ".join(fields)) + + def __str__(self): + """ + Returns a string representation of the choice when it is printed, + matching the Kconfig format (though without the contained choice + symbols). + + See Symbol.__str__() as well. + """ + return "\n".join(str(node) for node in self.nodes) + + # + # Private methods + # + + def __init__(self): + """ + Choice constructor -- not intended to be called directly by Kconfiglib + clients. + """ + # These attributes are always set on the instance from outside and + # don't need defaults: + # direct_dep + # kconfig + + self.orig_type = UNKNOWN + self.syms = [] + self.defaults = [] + + self.nodes = [] + + self.name = \ + self.user_value = self.user_selection = \ + self._cached_vis = self._cached_assignable = None + + self._cached_selection = _NO_CACHED_SELECTION + + # is_constant is checked by _make_depend_on(). Just set it to avoid + # having to special-case choices. + self.is_constant = self.is_optional = False + + # See Kconfig._build_dep() + self._dependents = set() + + # Used during dependency loop detection + self._checked = 0 + + def _assignable(self): + # Worker function for the 'assignable' attribute + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + vis = self.visibility + + if not vis: + return () + + if vis == 2: + if not self.is_optional: + return (2,) if self.type == BOOL else (1, 2) + return (0, 2) if self.type == BOOL else (0, 1, 2) + + # vis == 1 + + return (0, 1) if self.is_optional else (1,) + + def _selection(self): + # Worker function for the 'selection' attribute + + # Warning: See Symbol._rec_invalidate(), and note that this is a hidden + # function call (property magic) + if self.tri_value != 2: + # Not in y mode, so no selection + return None + + # Use the user selection if it's visible + if self.user_selection and self.user_selection.visibility: + return self.user_selection + + # Otherwise, check if we have a default + return self._get_selection_from_defaults() + + def _get_selection_from_defaults(self): + # Check if we have a default + for sym, cond in self.defaults: + # The default symbol must be visible too + if expr_value(cond) and sym.visibility: + return sym + + # Otherwise, pick the first visible symbol, if any + for sym in self.syms: + if sym.visibility: + return sym + + # Couldn't find a selection + return None + + def _invalidate(self): + self._cached_vis = self._cached_assignable = None + self._cached_selection = _NO_CACHED_SELECTION + + def _rec_invalidate(self): + # See Symbol._rec_invalidate() + + self._invalidate() + + for item in self._dependents: + if item._cached_vis is not None: + item._rec_invalidate() + +class MenuNode(object): + """ + Represents a menu node in the configuration. This corresponds to an entry + in e.g. the 'make menuconfig' interface, though non-visible choices, menus, + and comments also get menu nodes. If a symbol or choice is defined in + multiple locations, it gets one menu node for each location. + + The top-level menu node, corresponding to the implicit top-level menu, is + available in Kconfig.top_node. + + The menu nodes for a Symbol or Choice can be found in the + Symbol/Choice.nodes attribute. Menus and comments are represented as plain + menu nodes, with their text stored in the prompt attribute (prompt[0]). + This mirrors the C implementation. + + The following attributes are available on MenuNode instances. They should + be viewed as read-only. + + item: + Either a Symbol, a Choice, or one of the constants MENU and COMMENT. + Menus and comments are represented as plain menu nodes. Ifs are collapsed + (matching the C implementation) and do not appear in the final menu tree. + + next: + The following menu node. None if there is no following node. + + list: + The first child menu node. None if there are no children. + + Choices and menus naturally have children, but Symbols can also have + children because of menus created automatically from dependencies (see + kconfig-language.txt). + + parent: + The parent menu node. None if there is no parent. + + prompt: + A (string, cond) tuple with the prompt for the menu node and its + conditional expression (which is self.kconfig.y if there is no + condition). None if there is no prompt. + + For symbols and choices, the prompt is stored in the MenuNode rather than + the Symbol or Choice instance. For menus and comments, the prompt holds + the text. + + defaults: + The 'default' properties for this particular menu node. See + symbol.defaults. + + When evaluating defaults, you should use Symbol/Choice.defaults instead, + as it include properties from all menu nodes (a symbol/choice can have + multiple definition locations/menu nodes). MenuNode.defaults is meant for + documentation generation. + + selects: + Like MenuNode.defaults, for selects. + + implies: + Like MenuNode.defaults, for implies. + + ranges: + Like MenuNode.defaults, for ranges. + + help: + The help text for the menu node for Symbols and Choices. None if there is + no help text. Always stored in the node rather than the Symbol or Choice. + It is possible to have a separate help text at each location if a symbol + is defined in multiple locations. + + dep: + The 'depends on' dependencies for the menu node, or self.kconfig.y if + there are no dependencies. Parent dependencies are propagated to this + attribute, and this attribute is then in turn propagated to the + properties of symbols and choices. + + If a symbol or choice is defined in multiple locations, only the + properties defined at a particular location get the corresponding + MenuNode.dep dependencies propagated to them. + + visibility: + The 'visible if' dependencies for the menu node (which must represent a + menu), or self.kconfig.y if there are no 'visible if' dependencies. + 'visible if' dependencies are recursively propagated to the prompts of + symbols and choices within the menu. + + referenced: + A set() with all symbols and choices referenced in the properties and + property conditions of the menu node. + + Also includes dependencies inherited from surrounding menus and if's. + Choices appear in the dependencies of choice symbols. + + is_menuconfig: + Set to True if the children of the menu node should be displayed in a + separate menu. This is the case for the following items: + + - Menus (node.item == MENU) + + - Choices + + - Symbols defined with the 'menuconfig' keyword. The children come from + implicitly created submenus, and should be displayed in a separate + menu rather than being indented. + + 'is_menuconfig' is just a hint on how to display the menu node. It's + ignored internally by Kconfiglib, except when printing symbols. + + filename/linenr: + The location where the menu node appears. + + kconfig: + The Kconfig instance the menu node is from. + """ + __slots__ = ( + "dep", + "filename", + "help", + "is_menuconfig", + "item", + "kconfig", + "linenr", + "list", + "next", + "parent", + "prompt", + "visibility", + + # Properties + "defaults", + "selects", + "implies", + "ranges" + ) + + def __init__(self): + # Properties defined on this particular menu node. A local 'depends on' + # only applies to these, in case a symbol is defined in multiple + # locations. + self.defaults = [] + self.selects = [] + self.implies = [] + self.ranges = [] + + @property + def referenced(self): + """ + See the class documentation. + """ + # self.dep is included to catch dependencies from a lone 'depends on' + # when there are no properties to propagate it to + res = expr_items(self.dep) + + if self.prompt: + res |= expr_items(self.prompt[1]) + + if self.item == MENU: + res |= expr_items(self.visibility) + + for value, cond in self.defaults: + res |= expr_items(value) + res |= expr_items(cond) + + for value, cond in self.selects: + res.add(value) + res |= expr_items(cond) + + for value, cond in self.implies: + res.add(value) + res |= expr_items(cond) + + for low, high, cond in self.ranges: + res.add(low) + res.add(high) + res |= expr_items(cond) + + return res + + def __repr__(self): + """ + Returns a string with information about the menu node when it is + evaluated on e.g. the interactive Python prompt. + """ + fields = [] + + if isinstance(self.item, Symbol): + fields.append("menu node for symbol " + self.item.name) + + elif isinstance(self.item, Choice): + s = "menu node for choice" + if self.item.name is not None: + s += " " + self.item.name + fields.append(s) + + elif self.item == MENU: + fields.append("menu node for menu") + + elif self.item == COMMENT: + fields.append("menu node for comment") + + elif self.item is None: + fields.append("menu node for if (should not appear in the final " + " tree)") + + else: + _internal_error("unable to determine type in MenuNode.__repr__()") + + if self.prompt: + fields.append('prompt "{}" (visibility {})' + .format(self.prompt[0], + TRI_TO_STR[expr_value(self.prompt[1])])) + + if isinstance(self.item, Symbol) and self.is_menuconfig: + fields.append("is menuconfig") + + fields.append("deps " + TRI_TO_STR[expr_value(self.dep)]) + + if self.item == MENU: + fields.append("'visible if' deps " + \ + TRI_TO_STR[expr_value(self.visibility)]) + + if isinstance(self.item, (Symbol, Choice)) and self.help is not None: + fields.append("has help") + + if self.list: + fields.append("has child") + + if self.next: + fields.append("has next") + + fields.append("{}:{}".format(self.filename, self.linenr)) + + return "<{}>".format(", ".join(fields)) + + def __str__(self): + """ + Returns a string representation of the menu node, matching the Kconfig + format. + + The output could (almost) be fed back into a Kconfig parser to redefine + the object associated with the menu node. See the module documentation + for a gotcha related to choice symbols. + + For symbols and choices with multiple menu nodes (multiple definition + locations), properties that aren't associated with a particular menu + node are shown on all menu nodes ('option env=...', 'optional' for + choices, etc.). + """ + + return self._menu_comment_node_str() \ + if self.item in (MENU, COMMENT) else \ + self._sym_choice_node_str() + + def _menu_comment_node_str(self): + s = '{} "{}"\n'.format("menu" if self.item == MENU else "comment", + self.prompt[0]) + + if self.dep is not self.kconfig.y: + s += "\tdepends on {}\n".format(expr_str(self.dep)) + + if self.item == MENU and self.visibility is not self.kconfig.y: + s += "\tvisible if {}\n".format(expr_str(self.visibility)) + + return s + + def _sym_choice_node_str(self): + lines = [] + + def indent_add(s): + lines.append("\t" + s) + + def indent_add_cond(s, cond): + if cond is not self.kconfig.y: + s += " if " + expr_str(cond) + indent_add(s) + + if isinstance(self.item, (Symbol, Choice)): + sc = self.item + + if isinstance(sc, Symbol): + lines.append( + ("menuconfig " if self.is_menuconfig else "config ") + + sc.name) + else: + lines.append( + "choice" if sc.name is None else "choice " + sc.name) + + if sc.orig_type != UNKNOWN: + indent_add(TYPE_TO_STR[sc.orig_type]) + + if self.prompt: + indent_add_cond( + 'prompt "{}"'.format(escape(self.prompt[0])), + self.prompt[1]) + + if isinstance(sc, Symbol): + if sc.is_allnoconfig_y: + indent_add("option allnoconfig_y") + + if sc is sc.kconfig.defconfig_list: + indent_add("option defconfig_list") + + if sc.env_var is not None: + indent_add('option env="{}"'.format(sc.env_var)) + + if sc is sc.kconfig.modules: + indent_add("option modules") + + for low, high, cond in self.ranges: + indent_add_cond( + "range {} {}".format(expr_str(low), expr_str(high)), + cond) + + for default, cond in self.defaults: + indent_add_cond("default " + expr_str(default), cond) + + if isinstance(sc, Choice) and sc.is_optional: + indent_add("optional") + + if isinstance(sc, Symbol): + for select, cond in self.selects: + indent_add_cond("select " + expr_str(select), cond) + + for imply, cond in self.implies: + indent_add_cond("imply " + expr_str(imply), cond) + + if self.dep is not sc.kconfig.y: + indent_add("depends on " + expr_str(self.dep)) + + if self.help is not None: + indent_add("help") + for line in self.help.splitlines(): + indent_add(" " + line) + + return "\n".join(lines) + "\n" + +class Variable(object): + """ + Represents a preprocessor variable/function. + + The following attributes are available: + + name: + The name of the variable. + + value: + The unexpanded value of the variable. + + expanded_value: + The expanded value of the variable. For simple variables (those defined + with :=), this will equal 'value'. Accessing this property will raise a + KconfigError if any variable in the expansion expands to itself. + + is_recursive: + True if the variable is recursive (defined with =). + """ + __slots__ = ( + "_n_expansions", + "is_recursive", + "kconfig", + "name", + "value", + ) + + @property + def expanded_value(self): + """ + See the class documentation. + """ + return self.kconfig._expand_whole(self.value, ()) + +class KconfigError(Exception): + """ + Exception raised for Kconfig-related errors. + """ + +# Backwards compatibility +KconfigSyntaxError = KconfigError + +class InternalError(Exception): + """ + Exception raised for internal errors. + """ + +# +# Public functions +# + +def expr_value(expr): + """ + Evaluates the expression 'expr' to a tristate value. Returns 0 (n), 1 (m), + or 2 (y). + + 'expr' must be an already-parsed expression from a Symbol, Choice, or + MenuNode property. To evaluate an expression represented as a string, use + Kconfig.eval_string(). + + Passing subexpressions of expressions to this function works as expected. + """ + if not isinstance(expr, tuple): + return expr.tri_value + + if expr[0] == AND: + v1 = expr_value(expr[1]) + # Short-circuit the n case as an optimization (~5% faster + # allnoconfig.py and allyesconfig.py, as of writing) + return 0 if not v1 else min(v1, expr_value(expr[2])) + + if expr[0] == OR: + v1 = expr_value(expr[1]) + # Short-circuit the y case as an optimization + return 2 if v1 == 2 else max(v1, expr_value(expr[2])) + + if expr[0] == NOT: + return 2 - expr_value(expr[1]) + + if expr[0] in _RELATIONS: + # Implements <, <=, >, >= comparisons as well. These were added to + # kconfig in 31847b67 (kconfig: allow use of relations other than + # (in)equality). + + oper, op1, op2 = expr + + # If both operands are strings... + if op1.orig_type == STRING and op2.orig_type == STRING: + # ...then compare them lexicographically + comp = _strcmp(op1.str_value, op2.str_value) + else: + # Otherwise, try to compare them as numbers + try: + comp = _sym_to_num(op1) - _sym_to_num(op2) + except ValueError: + # Fall back on a lexicographic comparison if the operands don't + # parse as numbers + comp = _strcmp(op1.str_value, op2.str_value) + + if oper == EQUAL: res = comp == 0 + elif oper == UNEQUAL: res = comp != 0 + elif oper == LESS: res = comp < 0 + elif oper == LESS_EQUAL: res = comp <= 0 + elif oper == GREATER: res = comp > 0 + elif oper == GREATER_EQUAL: res = comp >= 0 + + return 2*res + + _internal_error("Internal error while evaluating expression: " + "unknown operation {}.".format(expr[0])) + +def expr_str(expr): + """ + Returns the string representation of the expression 'expr', as in a Kconfig + file. + + Passing subexpressions of expressions to this function works as expected. + """ + if isinstance(expr, Symbol): + if expr.is_constant: + return '"{}"'.format(escape(expr.name)) + return expr.name + + if isinstance(expr, Choice): + if expr.name is not None: + return "<choice {}>".format(expr.name) + return "<choice>" + + if expr[0] == NOT: + if isinstance(expr[1], tuple): + return "!({})".format(expr_str(expr[1])) + return "!" + expr_str(expr[1]) # Symbol + + if expr[0] == AND: + return "{} && {}".format(_parenthesize(expr[1], OR), + _parenthesize(expr[2], OR)) + + if expr[0] == OR: + # This turns A && B || C && D into "(A && B) || (C && D)", which is + # redundant, but more readable + return "{} || {}".format(_parenthesize(expr[1], AND), + _parenthesize(expr[2], AND)) + + # Relation + return "{} {} {}".format(expr_str(expr[1]), + _REL_TO_STR[expr[0]], + expr_str(expr[2])) + +def expr_items(expr): + """ + Returns a set() of all items (symbols and choices) that appear in the + expression 'expr'. + """ + + res = set() + + def rec(subexpr): + if isinstance(subexpr, tuple): + # AND, OR, NOT, or relation + + rec(subexpr[1]) + + # NOTs only have a single operand + if subexpr[0] != NOT: + rec(subexpr[2]) + + else: + # Symbol or choice + res.add(subexpr) + + rec(expr) + return res + +def split_expr(expr, op): + """ + Returns a list containing the top-level AND or OR operands in the + expression 'expr', in the same (left-to-right) order as they appear in + the expression. + + This can be handy e.g. for splitting (weak) reverse dependencies + from 'select' and 'imply' into individual selects/implies. + + op: + Either AND to get AND operands, or OR to get OR operands. + + (Having this as an operand might be more future-safe than having two + hardcoded functions.) + + + Pseudo-code examples: + + split_expr( A , OR ) -> [A] + split_expr( A && B , OR ) -> [A && B] + split_expr( A || B , OR ) -> [A, B] + split_expr( A || B , AND ) -> [A || B] + split_expr( A || B || (C && D) , OR ) -> [A, B, C && D] + + # Second || is not at the top level + split_expr( A || (B && (C || D)) , OR ) -> [A, B && (C || D)] + + # Parentheses don't matter as long as we stay at the top level (don't + # encounter any non-'op' nodes) + split_expr( (A || B) || C , OR ) -> [A, B, C] + split_expr( A || (B || C) , OR ) -> [A, B, C] + """ + res = [] + + def rec(subexpr): + if isinstance(subexpr, tuple) and subexpr[0] == op: + rec(subexpr[1]) + rec(subexpr[2]) + else: + res.append(subexpr) + + rec(expr) + return res + +def escape(s): + r""" + Escapes the string 's' in the same fashion as is done for display in + Kconfig format and when writing strings to a .config file. " and \ are + replaced by \" and \\, respectively. + """ + # \ must be escaped before " to avoid double escaping + return s.replace("\\", r"\\").replace('"', r'\"') + +# unescape() helper +_unescape_sub = re.compile(r"\\(.)").sub + +def unescape(s): + r""" + Unescapes the string 's'. \ followed by any character is replaced with just + that character. Used internally when reading .config files. + """ + return _unescape_sub(r"\1", s) + +def standard_kconfig(): + """ + Helper for tools. Loads the top-level Kconfig specified as the first + command-line argument, or "Kconfig" if there are no command-line arguments. + Returns the Kconfig instance. + + Exits with sys.exit() (which raises a SystemExit exception) and prints a + usage note to stderr if more than one command-line argument is passed. + """ + if len(sys.argv) > 2: + sys.exit("usage: {} [Kconfig]".format(sys.argv[0])) + + return Kconfig("Kconfig" if len(sys.argv) < 2 else sys.argv[1]) + +def standard_config_filename(): + """ + Helper for tools. Returns the value of KCONFIG_CONFIG (which specifies the + .config file to load/save) if it is set, and ".config" otherwise. + """ + return os.environ.get("KCONFIG_CONFIG", ".config") + +# +# Internal functions +# + +def _visibility(sc): + # Symbols and Choices have a "visibility" that acts as an upper bound on + # the values a user can set for them, corresponding to the visibility in + # e.g. 'make menuconfig'. This function calculates the visibility for the + # Symbol or Choice 'sc' -- the logic is nearly identical. + + vis = 0 + + for node in sc.nodes: + if node.prompt: + vis = max(vis, expr_value(node.prompt[1])) + + if isinstance(sc, Symbol) and sc.choice: + if sc.choice.orig_type == TRISTATE and sc.orig_type != TRISTATE and \ + sc.choice.tri_value != 2: + # Non-tristate choice symbols are only visible in y mode + return 0 + + if sc.orig_type == TRISTATE and vis == 1 and sc.choice.tri_value == 2: + # Choice symbols with m visibility are not visible in y mode + return 0 + + # Promote m to y if we're dealing with a non-tristate (possibly due to + # modules being disabled) + if vis == 1 and sc.type != TRISTATE: + return 2 + + return vis + +def _make_depend_on(sc, expr): + # Adds 'sc' (symbol or choice) as a "dependee" to all symbols in 'expr'. + # Constant symbols in 'expr' are skipped as they can never change value + # anyway. + + if isinstance(expr, tuple): + # AND, OR, NOT, or relation + + _make_depend_on(sc, expr[1]) + + # NOTs only have a single operand + if expr[0] != NOT: + _make_depend_on(sc, expr[2]) + + elif not expr.is_constant: + # Non-constant symbol, or choice + expr._dependents.add(sc) + +def _parenthesize(expr, type_): + # expr_str() helper. Adds parentheses around expressions of type 'type_'. + + if isinstance(expr, tuple) and expr[0] == type_: + return "({})".format(expr_str(expr)) + return expr_str(expr) + +def _indentation(line): + # Returns the length of the line's leading whitespace, treating tab stops + # as being spaced 8 characters apart. + + line = line.expandtabs() + return len(line) - len(line.lstrip()) + +def _is_base_n(s, n): + try: + int(s, n) + return True + except ValueError: + return False + +def _strcmp(s1, s2): + # strcmp()-alike that returns -1, 0, or 1 + + return (s1 > s2) - (s1 < s2) + +def _is_num(s): + # Returns True if the string 's' looks like a number. + # + # Internally, all operands in Kconfig are symbols, only undefined symbols + # (which numbers usually are) get their name as their value. + # + # Only hex numbers that start with 0x/0X are classified as numbers. + # Otherwise, symbols whose names happen to contain only the letters A-F + # would trigger false positives. + + try: + int(s) + except ValueError: + if not s.startswith(("0x", "0X")): + return False + + try: + int(s, 16) + except ValueError: + return False + + return True + +def _sym_to_num(sym): + # expr_value() helper for converting a symbol to a number. Raises + # ValueError for symbols that can't be converted. + + # For BOOL and TRISTATE, n/m/y count as 0/1/2. This mirrors 9059a3493ef + # ("kconfig: fix relational operators for bool and tristate symbols") in + # the C implementation. + return sym.tri_value if sym.orig_type in (BOOL, TRISTATE) else \ + int(sym.str_value, _TYPE_TO_BASE[sym.orig_type]) + +def _internal_error(msg): + raise InternalError( + msg + + "\nSorry! You may want to send an email to ulfalizer a.t Google's " + "email service to tell me about this. Include the message above and " + "the stack trace and describe what you were doing.") + +def _decoding_error(e, filename, macro_linenr=None): + # Gives the filename and context for UnicodeDecodeError's, which are a pain + # to debug otherwise. 'e' is the UnicodeDecodeError object. + # + # If the decoding error is for the output of a $(shell,...) command, + # macro_linenr holds the line number where it was run (the exact line + # number isn't available for decoding errors in files). + + if macro_linenr is None: + loc = filename + else: + loc = "output from macro at {}:{}".format(filename, macro_linenr) + + raise KconfigError( + "\n" + "Malformed {} in {}\n" + "Context: {}\n" + "Problematic data: {}\n" + "Reason: {}".format( + e.encoding, loc, + e.object[max(e.start - 40, 0):e.end + 40], + e.object[e.start:e.end], + e.reason)) + +def _name_and_loc(sc): + # Helper for giving the symbol/choice name and location(s) in e.g. warnings + + name = sc.name or "<choice>" + + if not sc.nodes: + return name + " (undefined)" + + return "{} (defined at {})".format( + name, + ", ".join("{}:{}".format(node.filename, node.linenr) + for node in sc.nodes)) + + +# Menu manipulation + +def _expr_depends_on(expr, sym): + # Reimplementation of expr_depends_symbol() from mconf.c. Used to determine + # if a submenu should be implicitly created. This also influences which + # items inside choice statements are considered choice items. + + if not isinstance(expr, tuple): + return expr is sym + + if expr[0] in (EQUAL, UNEQUAL): + # Check for one of the following: + # sym = m/y, m/y = sym, sym != n, n != sym + + left, right = expr[1:] + + if right is sym: + left, right = right, left + elif left is not sym: + return False + + return (expr[0] == EQUAL and right is sym.kconfig.m or \ + right is sym.kconfig.y) or \ + (expr[0] == UNEQUAL and right is sym.kconfig.n) + + return expr[0] == AND and \ + (_expr_depends_on(expr[1], sym) or + _expr_depends_on(expr[2], sym)) + +def _auto_menu_dep(node1, node2): + # Returns True if node2 has an "automatic menu dependency" on node1. If + # node2 has a prompt, we check its condition. Otherwise, we look directly + # at node2.dep. + + # If node2 has no prompt, use its menu node dependencies instead + return _expr_depends_on(node2.prompt[1] if node2.prompt else node2.dep, + node1.item) + +def _flatten(node): + # "Flattens" menu nodes without prompts (e.g. 'if' nodes and non-visible + # symbols with children from automatic menu creation) so that their + # children appear after them instead. This gives a clean menu structure + # with no unexpected "jumps" in the indentation. + + while node: + if node.list and not node.prompt: + last_node = node.list + while 1: + last_node.parent = node.parent + if not last_node.next: + break + last_node = last_node.next + + last_node.next = node.next + node.next = node.list + node.list = None + + node = node.next + +def _remove_ifs(node): + # Removes 'if' nodes (which can be recognized by MenuNode.item being None), + # which are assumed to already have been flattened. The C implementation + # doesn't bother to do this, but we expose the menu tree directly, and it + # makes it nicer to work with. + + first = node.list + while first and first.item is None: + first = first.next + + cur = first + while cur: + if cur.next and cur.next.item is None: + cur.next = cur.next.next + cur = cur.next + + node.list = first + +def _finalize_choice(node): + # Finalizes a choice, marking each symbol whose menu node has the choice as + # the parent as a choice symbol, and automatically determining types if not + # specified. + + choice = node.item + + cur = node.list + while cur: + if isinstance(cur.item, Symbol): + cur.item.choice = choice + choice.syms.append(cur.item) + cur = cur.next + + # If no type is specified for the choice, its type is that of + # the first choice item with a specified type + if choice.orig_type == UNKNOWN: + for item in choice.syms: + if item.orig_type != UNKNOWN: + choice.orig_type = item.orig_type + break + + # Each choice item of UNKNOWN type gets the type of the choice + for sym in choice.syms: + if sym.orig_type == UNKNOWN: + sym.orig_type = choice.orig_type + +def _check_dep_loop_sym(sym, ignore_choice): + # Detects dependency loops using depth-first search on the dependency graph + # (which is calculated earlier in Kconfig._build_dep()). + # + # Algorithm: + # + # 1. Symbols/choices start out with _checked = 0, meaning unvisited. + # + # 2. When a symbol/choice is first visited, _checked is set to 1, meaning + # "visited, potentially part of a dependency loop". The recursive + # search then continues from the symbol/choice. + # + # 3. If we run into a symbol/choice X with _checked already set to 1, + # there's a dependency loop. The loop is found on the call stack by + # recording symbols while returning ("on the way back") until X is seen + # again. + # + # 4. Once a symbol/choice and all its dependencies (or dependents in this + # case) have been checked recursively without detecting any loops, its + # _checked is set to 2, meaning "visited, not part of a dependency + # loop". + # + # This saves work if we run into the symbol/choice again in later calls + # to _check_dep_loop_sym(). We just return immediately. + # + # Choices complicate things, as every choice symbol depends on every other + # choice symbol in a sense. When a choice is "entered" via a choice symbol + # X, we visit all choice symbols from the choice except X, and prevent + # immediately revisiting the choice with a flag (ignore_choice). + # + # Maybe there's a better way to handle this (different flags or the + # like...) + + if not sym._checked: + # sym._checked == 0, unvisited + + sym._checked = 1 + + for dep in sym._dependents: + # Choices show up in Symbol._dependents when the choice has the + # symbol in a 'prompt' or 'default' condition (e.g. + # 'default ... if SYM'). + # + # Since we aren't entering the choice via a choice symbol, all + # choice symbols need to be checked, hence the None. + loop = _check_dep_loop_choice(dep, None) \ + if isinstance(dep, Choice) \ + else _check_dep_loop_sym(dep, False) + + if loop: + # Dependency loop found + return _found_dep_loop(loop, sym) + + if sym.choice and not ignore_choice: + loop = _check_dep_loop_choice(sym.choice, sym) + if loop: + # Dependency loop found + return _found_dep_loop(loop, sym) + + # The symbol is not part of a dependency loop + sym._checked = 2 + + # No dependency loop found + return None + + if sym._checked == 2: + # The symbol was checked earlier and is already known to not be part of + # a dependency loop + return None + + # sym._checked == 1, found a dependency loop. Return the symbol as the + # first element in it. + return (sym,) + +def _check_dep_loop_choice(choice, skip): + if not choice._checked: + # choice._checked == 0, unvisited + + choice._checked = 1 + + # Check for loops involving choice symbols. If we came here via a + # choice symbol, skip that one, as we'd get a false positive + # '<sym FOO> -> <choice> -> <sym FOO>' loop otherwise. + for sym in choice.syms: + if sym is not skip: + # Prevent the choice from being immediately re-entered via the + # "is a choice symbol" path by passing True + loop = _check_dep_loop_sym(sym, True) + if loop: + # Dependency loop found + return _found_dep_loop(loop, choice) + + # The choice is not part of a dependency loop + choice._checked = 2 + + # No dependency loop found + return None + + if choice._checked == 2: + # The choice was checked earlier and is already known to not be part of + # a dependency loop + return None + + # choice._checked == 1, found a dependency loop. Return the choice as the + # first element in it. + return (choice,) + +def _found_dep_loop(loop, cur): + # Called "on the way back" when we know we have a loop + + # Is the symbol/choice 'cur' where the loop started? + if cur is not loop[0]: + # Nope, it's just a part of the loop + return loop + (cur,) + + # Yep, we have the entire loop. Throw an exception that shows it. + + msg = "\nDependency loop\n" \ + "===============\n\n" + + for item in loop: + if item is not loop[0]: + msg += "...depends on " + if isinstance(item, Symbol) and item.choice: + msg += "the choice symbol " + + msg += "{}, with definition...\n\n{}\n" \ + .format(_name_and_loc(item), item) + + # Small wart: Since we reuse the already calculated + # Symbol/Choice._dependents sets for recursive dependency detection, we + # lose information on whether a dependency came from a 'select'/'imply' + # condition or e.g. a 'depends on'. + # + # This might cause selecting symbols to "disappear". For example, + # a symbol B having 'select A if C' gives a direct dependency from A to + # C, since it corresponds to a reverse dependency of B && C. + # + # Always print reverse dependencies for symbols that have them to make + # sure information isn't lost. I wonder if there's some neat way to + # improve this. + + if isinstance(item, Symbol): + if item.rev_dep is not item.kconfig.n: + msg += "(select-related dependencies: {})\n\n" \ + .format(expr_str(item.rev_dep)) + + if item.weak_rev_dep is not item.kconfig.n: + msg += "(imply-related dependencies: {})\n\n" \ + .format(expr_str(item.rev_dep)) + + msg += "...depends again on {}".format(_name_and_loc(loop[0])) + + raise KconfigError(msg) + +def _check_sym_sanity(sym): + # Checks various symbol properties that are handiest to check after + # parsing. Only generates errors and warnings. + + if sym.orig_type in (BOOL, TRISTATE): + # A helper function could be factored out here, but keep it + # speedy/straightforward for now. bool/tristate symbols are by far the + # most common, and most lack selects and implies. + + for target_sym, _ in sym.selects: + if target_sym.orig_type not in (BOOL, TRISTATE, UNKNOWN): + sym.kconfig._warn("{} selects the {} symbol {}, which is not " + "bool or tristate" + .format(_name_and_loc(sym), + TYPE_TO_STR[target_sym.orig_type], + _name_and_loc(target_sym))) + + for target_sym, _ in sym.implies: + if target_sym.orig_type not in (BOOL, TRISTATE, UNKNOWN): + sym.kconfig._warn("{} implies the {} symbol {}, which is not " + "bool or tristate" + .format(_name_and_loc(sym), + TYPE_TO_STR[target_sym.orig_type], + _name_and_loc(target_sym))) + + elif sym.orig_type in (STRING, INT, HEX): + for default, _ in sym.defaults: + if not isinstance(default, Symbol): + raise KconfigError( + "the {} symbol {} has a malformed default {} -- expected " + "a single symbol" + .format(TYPE_TO_STR[sym.orig_type], _name_and_loc(sym), + expr_str(default))) + + if sym.orig_type == STRING: + if not default.is_constant and not default.nodes and \ + not default.name.isupper(): + # 'default foo' on a string symbol could be either a symbol + # reference or someone leaving out the quotes. Guess that + # the quotes were left out if 'foo' isn't all-uppercase + # (and no symbol named 'foo' exists). + sym.kconfig._warn("style: quotes recommended around " + "default value for string symbol " + + _name_and_loc(sym)) + + elif sym.orig_type in (INT, HEX) and \ + not _int_hex_ok(default, sym.orig_type): + + sym.kconfig._warn("the {0} symbol {1} has a non-{0} default {2}" + .format(TYPE_TO_STR[sym.orig_type], + _name_and_loc(sym), + _name_and_loc(default))) + + if sym.selects or sym.implies: + sym.kconfig._warn("the {} symbol {} has selects or implies" + .format(TYPE_TO_STR[sym.orig_type], + _name_and_loc(sym))) + + else: # UNKNOWN + sym.kconfig._warn("{} defined without a type" + .format(_name_and_loc(sym))) + + + if sym.ranges: + if sym.orig_type not in (INT, HEX): + sym.kconfig._warn( + "the {} symbol {} has ranges, but is not int or hex" + .format(TYPE_TO_STR[sym.orig_type], _name_and_loc(sym))) + else: + for low, high, _ in sym.ranges: + if not _int_hex_ok(low, sym.orig_type) or \ + not _int_hex_ok(high, sym.orig_type): + + sym.kconfig._warn("the {0} symbol {1} has a non-{0} range " + "[{2}, {3}]" + .format(TYPE_TO_STR[sym.orig_type], + _name_and_loc(sym), + _name_and_loc(low), + _name_and_loc(high))) + + +def _int_hex_ok(sym, type_): + # Returns True if the (possibly constant) symbol 'sym' is valid as a value + # for a symbol of type type_ (INT or HEX) + + # 'not sym.nodes' implies a constant or undefined symbol, e.g. a plain + # "123" + if not sym.nodes: + return _is_base_n(sym.name, _TYPE_TO_BASE[type_]) + + return sym.orig_type == type_ + +def _check_choice_sanity(choice): + # Checks various choice properties that are handiest to check after + # parsing. Only generates errors and warnings. + + if choice.orig_type not in (BOOL, TRISTATE): + choice.kconfig._warn("{} defined with type {}" + .format(_name_and_loc(choice), + TYPE_TO_STR[choice.orig_type])) + + for node in choice.nodes: + if node.prompt: + break + else: + choice.kconfig._warn(_name_and_loc(choice) + + " defined without a prompt") + + for default, _ in choice.defaults: + if not isinstance(default, Symbol): + raise KconfigError( + "{} has a malformed default {}" + .format(_name_and_loc(choice), expr_str(default))) + + if default.choice is not choice: + choice.kconfig._warn("the default selection {} of {} is not " + "contained in the choice" + .format(_name_and_loc(default), + _name_and_loc(choice))) + + for sym in choice.syms: + if sym.defaults: + sym.kconfig._warn("default on the choice symbol {} will have " + "no effect".format(_name_and_loc(sym))) + + if sym.rev_dep is not sym.kconfig.n: + _warn_choice_select_imply(sym, sym.rev_dep, "selected") + + if sym.weak_rev_dep is not sym.kconfig.n: + _warn_choice_select_imply(sym, sym.weak_rev_dep, "implied") + + for node in sym.nodes: + if node.parent.item is choice: + if not node.prompt: + sym.kconfig._warn("the choice symbol {} has no prompt" + .format(_name_and_loc(sym))) + + elif node.prompt: + sym.kconfig._warn("the choice symbol {} is defined with a " + "prompt outside the choice" + .format(_name_and_loc(sym))) + +def _warn_choice_select_imply(sym, expr, expr_type): + msg = "the choice symbol {} is {} by the following symbols, which has " \ + "no effect: ".format(_name_and_loc(sym), expr_type) + + # si = select/imply + for si in split_expr(expr, OR): + msg += "\n - " + _name_and_loc(split_expr(si, AND)[0]) + + sym.kconfig._warn(msg) + +# Predefined preprocessor functions + +def _filename_fn(kconf, args): + return kconf._filename + +def _lineno_fn(kconf, args): + return str(kconf._linenr) + +def _info_fn(kconf, args): + print("{}:{}: {}".format(kconf._filename, kconf._linenr, args[1])) + + return "" + +def _warning_if_fn(kconf, args): + if args[1] == "y": + kconf._warn(args[2], kconf._filename, kconf._linenr) + + return "" + +def _error_if_fn(kconf, args): + if args[1] == "y": + raise KconfigError("{}:{}: {}".format( + kconf._filename, kconf._linenr, args[2])) + + return "" + +def _shell_fn(kconf, args): + # Use universal newlines mode to prevent e.g. stray \r's in command output + # on Windows + + if _IS_PY2: + # No decoding on Python 2 + stdout, stderr = subprocess.Popen( + args[1], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True + ).communicate() + + else: + # Passing universal_newlines=True and/or 'encoding' on Python 3 turns + # on decoding of the output (bytes -> str), which might fail + try: + stdout, stderr = subprocess.Popen( + args[1], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, encoding=kconf._encoding + ).communicate() + except UnicodeDecodeError as e: + _decoding_error(e, kconf._filename, kconf._linenr) + + if stderr: + kconf._warn( + "'{}' wrote to stderr: {}".format(args[1], stderr.rstrip("\n")), + kconf._filename, kconf._linenr) + + return stdout.rstrip("\n").replace("\n", " ") + +# +# Public global constants +# + +# Integers representing symbol types +( + BOOL, + HEX, + INT, + STRING, + TRISTATE, + UNKNOWN +) = range(6) + +# Integers representing menu and comment menu nodes +( + MENU, + COMMENT, +) = range(2) + +# Converts a symbol/choice type to a string +TYPE_TO_STR = { + UNKNOWN: "unknown", + BOOL: "bool", + TRISTATE: "tristate", + STRING: "string", + HEX: "hex", + INT: "int", +} + +TRI_TO_STR = { + 0: "n", + 1: "m", + 2: "y", +} + +STR_TO_TRI = { + "n": 0, + "m": 1, + "y": 2, +} + +# +# Internal global constants (plus public expression type +# constants) +# + +# Are we running on Python 2? +_IS_PY2 = sys.version_info[0] < 3 + +# Tokens, with values 1, 2, ... . Avoiding 0 simplifies some checks by making +# all tokens except empty strings truthy. +( + _T_ALLNOCONFIG_Y, + _T_AND, + _T_BOOL, + _T_CHOICE, + _T_CLOSE_PAREN, + _T_COMMENT, + _T_CONFIG, + _T_DEFAULT, + _T_DEFCONFIG_LIST, + _T_DEF_BOOL, + _T_DEF_HEX, + _T_DEF_INT, + _T_DEF_STRING, + _T_DEF_TRISTATE, + _T_DEPENDS, + _T_ENDCHOICE, + _T_ENDIF, + _T_ENDMENU, + _T_ENV, + _T_EQUAL, + _T_GREATER, + _T_GREATER_EQUAL, + _T_HELP, + _T_HEX, + _T_IF, + _T_IMPLY, + _T_INT, + _T_LESS, + _T_LESS_EQUAL, + _T_MAINMENU, + _T_MENU, + _T_MENUCONFIG, + _T_MODULES, + _T_NOT, + _T_ON, + _T_OPEN_PAREN, + _T_OPTION, + _T_OPTIONAL, + _T_OR, + _T_ORSOURCE, + _T_OSOURCE, + _T_PROMPT, + _T_RANGE, + _T_RSOURCE, + _T_SELECT, + _T_SOURCE, + _T_STRING, + _T_TRISTATE, + _T_UNEQUAL, + _T_VISIBLE, +) = range(1, 51) + +# Public integers representing expression types +# +# Having these match the value of the corresponding tokens removes the need +# for conversion +AND = _T_AND +OR = _T_OR +NOT = _T_NOT +EQUAL = _T_EQUAL +UNEQUAL = _T_UNEQUAL +LESS = _T_LESS +LESS_EQUAL = _T_LESS_EQUAL +GREATER = _T_GREATER +GREATER_EQUAL = _T_GREATER_EQUAL + +# Keyword to token map, with the get() method assigned directly as a small +# optimization +_get_keyword = { + "---help---": _T_HELP, + "allnoconfig_y": _T_ALLNOCONFIG_Y, + "bool": _T_BOOL, + "boolean": _T_BOOL, + "choice": _T_CHOICE, + "comment": _T_COMMENT, + "config": _T_CONFIG, + "def_bool": _T_DEF_BOOL, + "def_hex": _T_DEF_HEX, + "def_int": _T_DEF_INT, + "def_string": _T_DEF_STRING, + "def_tristate": _T_DEF_TRISTATE, + "default": _T_DEFAULT, + "defconfig_list": _T_DEFCONFIG_LIST, + "depends": _T_DEPENDS, + "endchoice": _T_ENDCHOICE, + "endif": _T_ENDIF, + "endmenu": _T_ENDMENU, + "env": _T_ENV, + "grsource": _T_ORSOURCE, # Backwards compatibility + "gsource": _T_OSOURCE, # Backwards compatibility + "help": _T_HELP, + "hex": _T_HEX, + "if": _T_IF, + "imply": _T_IMPLY, + "int": _T_INT, + "mainmenu": _T_MAINMENU, + "menu": _T_MENU, + "menuconfig": _T_MENUCONFIG, + "modules": _T_MODULES, + "on": _T_ON, + "option": _T_OPTION, + "optional": _T_OPTIONAL, + "orsource": _T_ORSOURCE, + "osource": _T_OSOURCE, + "prompt": _T_PROMPT, + "range": _T_RANGE, + "rsource": _T_RSOURCE, + "select": _T_SELECT, + "source": _T_SOURCE, + "string": _T_STRING, + "tristate": _T_TRISTATE, + "visible": _T_VISIBLE, +}.get + +# Tokens after which strings are expected. This is used to tell strings from +# constant symbol references during tokenization, both of which are enclosed in +# quotes. +# +# Identifier-like lexemes ("missing quotes") are also treated as strings after +# these tokens. _T_CHOICE is included to avoid symbols being registered for +# named choices. +_STRING_LEX = frozenset(( + _T_BOOL, + _T_CHOICE, + _T_COMMENT, + _T_HEX, + _T_INT, + _T_MAINMENU, + _T_MENU, + _T_ORSOURCE, + _T_OSOURCE, + _T_PROMPT, + _T_RSOURCE, + _T_SOURCE, + _T_STRING, + _T_TRISTATE, +)) + +# Tokens for types, excluding def_bool, def_tristate, etc., for quick +# checks during parsing +_TYPE_TOKENS = frozenset(( + _T_BOOL, + _T_TRISTATE, + _T_INT, + _T_HEX, + _T_STRING, +)) + + +# Helper functions for getting compiled regular expressions, with the needed +# matching function returned directly as a small optimization. +# +# Use ASCII regex matching on Python 3. It's already the default on Python 2. + +def _re_match(regex): + return re.compile(regex, 0 if _IS_PY2 else re.ASCII).match + +def _re_search(regex): + return re.compile(regex, 0 if _IS_PY2 else re.ASCII).search + + +# Various regular expressions used during parsing + +# The initial token on a line. Also eats leading and trailing whitespace, so +# that we can jump straight to the next token (or to the end of the line if +# there is only one token). +# +# This regex will also fail to match for empty lines and comment lines. +# +# '$' is included to detect a variable assignment left-hand side with a $ in it +# (which might be from a macro expansion). +_command_match = _re_match(r"\s*([$A-Za-z0-9_-]+)\s*") + +# An identifier/keyword after the first token. Also eats trailing whitespace. +_id_keyword_match = _re_match(r"([A-Za-z0-9_/.-]+)\s*") + +# A fragment in the left-hand side of a preprocessor variable assignment. These +# are the portions between macro expansions ($(foo)). Macros are supported in +# the LHS (variable name). +_assignment_lhs_fragment_match = _re_match("[A-Za-z0-9_-]*") + +# The assignment operator and value (right-hand side) in a preprocessor +# variable assignment +_assignment_rhs_match = _re_match(r"\s*(=|:=|\+=)\s*(.*)") + +# Special characters/strings while expanding a macro (')', ',', and '$(') +_macro_special_search = _re_search(r"\)|,|\$\(") + +# Special characters/strings while expanding a string (quotes, '\', and '$(') +_string_special_search = _re_search(r'"|\'|\\|\$\(') + +# A valid right-hand side for an assignment to a string symbol in a .config +# file, including escaped characters. Extracts the contents. +_conf_string_match = _re_match(r'"((?:[^\\"]|\\.)*)"') + + +# Token to type mapping +_TOKEN_TO_TYPE = { + _T_BOOL: BOOL, + _T_DEF_BOOL: BOOL, + _T_DEF_HEX: HEX, + _T_DEF_INT: INT, + _T_DEF_STRING: STRING, + _T_DEF_TRISTATE: TRISTATE, + _T_HEX: HEX, + _T_INT: INT, + _T_STRING: STRING, + _T_TRISTATE: TRISTATE, +} + +# Constant representing that there's no cached choice selection. This is +# distinct from a cached None (no selection). We create a unique object (any +# will do) for it so we can test with 'is'. +_NO_CACHED_SELECTION = object() + +# Used in comparisons. 0 means the base is inferred from the format of the +# string. +_TYPE_TO_BASE = { + HEX: 16, + INT: 10, + STRING: 0, + UNKNOWN: 0, +} + +# Note: These constants deliberately equal the corresponding tokens (_T_EQUAL, +# _T_UNEQUAL, etc.), which removes the need for conversion +_RELATIONS = frozenset(( + EQUAL, + UNEQUAL, + LESS, + LESS_EQUAL, + GREATER, + GREATER_EQUAL, +)) + +_REL_TO_STR = { + EQUAL: "=", + UNEQUAL: "!=", + LESS: "<", + LESS_EQUAL: "<=", + GREATER: ">", + GREATER_EQUAL: ">=", +} diff --git a/scripts/make.inc.sh b/scripts/make.inc.sh new file mode 100644 index 0000000..598e306 --- /dev/null +++ b/scripts/make.inc.sh @@ -0,0 +1,58 @@ +# intended to be included from shell-script + +PORT=${PORT:-/dev/ttyUSB0} +_PROJECT_DIR=${0%/*} +_SCRIPTS_DIR=${0%/*}/../scripts + +if test -z "$PY_SOURCES"; then + echo "PY_SOURCES variable must be set" + exit 1 +fi + +testport() { + if test -c $PORT; then + true + else + echo "$PORT not connected" >&2 + exit 1 + fi +} + +case "$1" in + console) + testport + screen $PORT 115200 + ;; + flash) + testport + $_SCRIPTS_DIR/flash.sh "${PORT}" + ;; + setup) + testport + $_SCRIPTS_DIR/setup_wlan.sh "${PORT}" "${_PROJECT_DIR}/.config" + ;; + copy) + testport + for file in $PY_SOURCES; do + $_SCRIPTS_DIR/transfer.sh "${PORT}" "${_PROJECT_DIR}/$file" "$file" + done + ;; + format) + testport + $_SCRIPTS_DIR/format.sh "${PORT}" + ;; + reset) + testport + echo -e -n "\003\004" > "${PORT}" + sleep 3 + ;; + config) + export srctree="${_PROJECT_DIR}" + export KCONFIG_CONFIG="${_PROJECT_DIR}/.config" + python3 "$_SCRIPTS_DIR/menuconfig.py" "${_PROJECT_DIR}/Kconfig" + ;; + *) + echo "Command '$1' not supported" >&2 + exit 1 + ;; +esac diff --git a/scripts/menuconfig.py b/scripts/menuconfig.py new file mode 100644 index 0000000..a805610 --- /dev/null +++ b/scripts/menuconfig.py @@ -0,0 +1,2454 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2018, Nordic Semiconductor ASA and Ulf Magnusson +# SPDX-License-Identifier: ISC + +""" +Overview +======== + +A curses-based menuconfig implementation. The interface should feel familiar to +people used to mconf ('make menuconfig'). + +Supports the same keys as mconf, and also supports a set of keybindings +inspired by Vi: + + J/K : Down/Up + L : Enter menu/Toggle item + H : Leave menu + Ctrl-D/U: Page Down/Page Down + G/End : Jump to end of list + g/Home : Jump to beginning of list + +The mconf feature where pressing a key jumps to a menu entry with that +character in it in the current menu isn't supported. A jump-to feature for +jumping directly to any symbol (including invisible symbols) is available +instead. + +Space and Enter are "smart" and try to do what you'd expect for the given +menu entry. + + +Running +======= + +menuconfig.py can be run either as a standalone executable or by calling the +menu.menuconfig() function with an existing Kconfig instance. The second option +is a bit inflexible in that it will still load and save .config, etc. + +When run in standalone mode, the top-level Kconfig file to load can be passed +as a command-line argument. With no argument, it defaults to "Kconfig". + +The KCONFIG_CONFIG environment variable specifies the .config file to load (if +it exists) and save. If KCONFIG_CONFIG is unset, ".config" is used. + +$srctree is supported through Kconfiglib. + + +Other features +============== + + - Seamless terminal resizing + + - No dependencies on *nix, as the 'curses' module is in the Python standard + library + + - Unicode text entry + + - Improved information screen compared to mconf: + + * Expressions are split up by their top-level &&/|| operands to improve + readability + + * Undefined symbols in expressions are pointed out + + * Menus and comments have information displays + + * Kconfig definitions are printed + + +Limitations +=========== + + - Python 3 only + + This is mostly due to Python 2 not having curses.get_wch(), which is needed + for Unicode support. + + - Doesn't work out of the box on Windows + + Has been tested to work with the wheels provided at + https://www.lfd.uci.edu/~gohlke/pythonlibs/#curses though. +""" + +import curses +import errno +import locale +import os +import platform +import re +import textwrap + +# We need this double import for the _expr_str() override below +import kconfiglib + +from kconfiglib import Symbol, Choice, MENU, COMMENT, MenuNode, \ + BOOL, STRING, INT, HEX, UNKNOWN, \ + AND, OR, NOT, \ + expr_value, split_expr, \ + TRI_TO_STR, TYPE_TO_STR, \ + standard_kconfig, standard_config_filename + + +# +# Configuration variables +# + +# If True, try to convert LC_CTYPE to a UTF-8 locale if it is set to the C +# locale (which implies ASCII). This fixes curses Unicode I/O issues on systems +# with bad defaults. ncurses configures itself from the locale settings. +# +# Related PEP: https://www.python.org/dev/peps/pep-0538/ +_CONVERT_C_LC_CTYPE_TO_UTF8 = True + +# How many steps an implicit submenu will be indented. Implicit submenus are +# created when an item depends on the symbol before it. Note that symbols +# defined with 'menuconfig' create a separate menu instead of indenting. +_SUBMENU_INDENT = 4 + +# Number of steps for Page Up/Down to jump +_PG_JUMP = 6 + +# How far the cursor needs to be from the edge of the window before it starts +# to scroll. Used for the main menu display, the information display, the +# search display, and for text boxes. +_SCROLL_OFFSET = 5 + +# Minimum width of dialogs that ask for text input +_INPUT_DIALOG_MIN_WIDTH = 30 + +# Number of arrows pointing up/down to draw when a window is scrolled +_N_SCROLL_ARROWS = 14 + +# Lines of help text shown at the bottom of the "main" display +_MAIN_HELP_LINES = """ +[Space/Enter] Toggle/enter [ESC] Leave menu [S] Save +[O] Load [?] Symbol info [/] Jump to symbol +[A] Toggle show-all mode [C] Toggle show-name mode +[Q] Quit (prompts for save) [D] Save minimal config (advanced) +"""[1:-1].split("\n") + +# Lines of help text shown at the bottom of the information dialog +_INFO_HELP_LINES = """ +[ESC/q] Return to menu +"""[1:-1].split("\n") + +# Lines of help text shown at the bottom of the search dialog +_JUMP_TO_HELP_LINES = """ +Type text to narrow the search. Regexes are supported (via Python's 're' +module). The up/down cursor keys step in the list. [Enter] jumps to the +selected symbol. [ESC] aborts the search. Type multiple space-separated +strings/regexes to find entries that match all of them. Type Ctrl-F to +view the help of the selected item without leaving the dialog. +"""[1:-1].split("\n") + +def _init_styles(): + global _SEPARATOR_STYLE + global _HELP_STYLE + global _LIST_STYLE + global _LIST_SEL_STYLE + global _LIST_INVISIBLE_STYLE + global _LIST_INVISIBLE_SEL_STYLE + global _INPUT_FIELD_STYLE + + global _PATH_STYLE + + global _DIALOG_FRAME_STYLE + global _DIALOG_BODY_STYLE + + global _INFO_TEXT_STYLE + + # Initialize styles for different parts of the application. The arguments + # are ordered as follows: + # + # 1. Text color + # 2. Background color + # 3. Attributes + # 4. Extra attributes if colors aren't available. The colors will be + # ignored in this case, and the attributes from (3.) and (4.) will be + # ORed together. + + # A_BOLD tends to produce faint and hard-to-read text on the Windows + # console, especially with the old color scheme, before the introduction of + # https://blogs.msdn.microsoft.com/commandline/2017/08/02/updating-the-windows-console-colors/ + BOLD = curses.A_NORMAL if _IS_WINDOWS else curses.A_BOLD + + + # Separator lines between windows. Also used for the top line in the symbol + # information dialog. + _SEPARATOR_STYLE = _style(curses.COLOR_BLACK, curses.COLOR_YELLOW, BOLD, curses.A_STANDOUT) + + # Edit boxes + _INPUT_FIELD_STYLE = _style(curses.COLOR_WHITE, curses.COLOR_BLUE, curses.A_NORMAL, curses.A_STANDOUT) + + # List of items, e.g. the main display + _LIST_STYLE = _style(curses.COLOR_BLACK, curses.COLOR_WHITE, curses.A_NORMAL ) + # Style for the selected item + _LIST_SEL_STYLE = _style(curses.COLOR_WHITE, curses.COLOR_BLUE, curses.A_NORMAL, curses.A_STANDOUT) + + # Like _LIST_(SEL_)STYLE, for invisible items. Used in show-all mode. + _LIST_INVISIBLE_STYLE = _style(curses.COLOR_RED, curses.COLOR_WHITE, curses.A_NORMAL, BOLD ) + _LIST_INVISIBLE_SEL_STYLE = _style(curses.COLOR_RED, curses.COLOR_BLUE, curses.A_NORMAL, curses.A_STANDOUT) + + # Help text windows at the bottom of various fullscreen dialogs + _HELP_STYLE = _style(curses.COLOR_BLACK, curses.COLOR_WHITE, BOLD ) + + # Top row in the main display, with the menu path + _PATH_STYLE = _style(curses.COLOR_BLACK, curses.COLOR_WHITE, BOLD ) + + # Symbol information text + _INFO_TEXT_STYLE = _LIST_STYLE + + # Frame around dialog boxes + _DIALOG_FRAME_STYLE = _style(curses.COLOR_BLACK, curses.COLOR_YELLOW, BOLD, curses.A_STANDOUT) + # Body of dialog boxes + _DIALOG_BODY_STYLE = _style(curses.COLOR_WHITE, curses.COLOR_BLACK, curses.A_NORMAL ) + + +# +# Main application +# + +# color_attribs holds the color pairs we've already created, indexed by a +# (<foreground color>, <background color>) tuple. +# +# Obscure Python: We never pass a value for color_attribs, and it keeps +# pointing to the same dict. This avoids a global. +def _style(fg_color, bg_color, attribs, no_color_extra_attribs=0, + color_attribs={}): + # Returns an attribute with the specified foreground and background color + # and the attributes in 'attribs'. Reuses color pairs already created if + # possible, and creates a new color pair otherwise. + # + # Returns 'attribs | no_color_extra_attribs' if colors aren't supported. + + if not curses.has_colors(): + return attribs | no_color_extra_attribs + + if (fg_color, bg_color) not in color_attribs: + # Create new color pair. Color pair number 0 is hardcoded and cannot be + # changed, hence the +1s. + curses.init_pair(len(color_attribs) + 1, fg_color, bg_color) + color_attribs[(fg_color, bg_color)] = \ + curses.color_pair(len(color_attribs) + 1) + + return color_attribs[(fg_color, bg_color)] | attribs + +# "Extend" the standard kconfiglib.expr_str() to show values for symbols +# appearing in expressions, for the information dialog. +# +# This is a bit hacky, but officially supported. It beats having to reimplement +# expression printing just to tweak it a bit. + +def _expr_str_val(expr): + if isinstance(expr, Symbol) and not expr.is_constant and \ + not _is_num(expr.name): + # Show the values of non-constant (non-quoted) symbols that don't look + # like numbers. Things like 123 are actually a symbol references, and + # only work as expected due to undefined symbols getting their name as + # their value. Showing the symbol value there isn't helpful though. + + if not expr.nodes: + # Undefined symbol reference + return "{}(undefined/n)".format(expr.name) + + return '{}(="{}")'.format(expr.name, expr.str_value) + + if isinstance(expr, tuple) and expr[0] == NOT and \ + isinstance(expr[1], Symbol): + + # Put a space after "!" before a symbol, since '! FOO(="y")' makes it + # clearer than '!FOO(="y")' that "y" is the value of FOO itself + return "! " + _expr_str(expr[1]) + + # We'll end up back in _expr_str_val() when _expr_str_orig() does recursive + # calls for subexpressions + return _expr_str_orig(expr) + +# Do hacky expr_str() extension. The rest of the code will just call +# _expr_str(). +_expr_str_orig = kconfiglib.expr_str +kconfiglib.expr_str = _expr_str_val +_expr_str = _expr_str_val + +# Entry point when run as an executable, split out so that setuptools' +# 'entry_points' can be used. It produces a handy menuconfig.exe launcher on +# Windows. +def _main(): + menuconfig(standard_kconfig()) + +def menuconfig(kconf): + """ + Launches the configuration interface, returning after the user exits. + + kconf: + Kconfig instance to be configured + """ + + globals()["_kconf"] = kconf + global _config_filename + global _show_all + global _conf_changed + + + _config_filename = standard_config_filename() + + if os.path.exists(_config_filename): + _conf_changed = False + print("Using existing configuration '{}' as base" + .format(_config_filename)) + _kconf.load_config(_config_filename) + + else: + # Always prompt for save if the output configuration file doesn't exist + _conf_changed = True + + if kconf.defconfig_filename is not None: + print("Using default configuration found in '{}' as base" + .format(kconf.defconfig_filename)) + _kconf.load_config(kconf.defconfig_filename) + + else: + print("Using default symbol values as base") + + + # Any visible items in the top menu? + _show_all = False + if not _shown_nodes(_kconf.top_node): + # Nothing visible. Start in show-all mode and try again. + _show_all = True + if not _shown_nodes(_kconf.top_node): + # Give up. The implementation relies on always having a selected + # node. + print("Empty configuration -- nothing to configure.\n" + "Check that environment variables are set properly.") + return + + # Disable warnings. They get mangled in curses mode, and we deal with + # errors ourselves. + _kconf.disable_warnings() + + # Make curses use the locale settings specified in the environment + locale.setlocale(locale.LC_ALL, "") + + # Try to fix Unicode issues on systems with bad defaults + if _CONVERT_C_LC_CTYPE_TO_UTF8: + _convert_c_lc_ctype_to_utf8() + + # Get rid of the delay between pressing ESC and jumping to the parent menu + os.environ.setdefault("ESCDELAY", "0") + + # Enter curses mode. _menuconfig() returns a string to print on exit, after + # curses has been de-initialized. + print(curses.wrapper(_menuconfig)) + +# Global variables used below: +# +# _cur_menu: +# Menu node of the menu (or menuconfig symbol, or choice) currently being +# shown +# +# _shown: +# List of items in _cur_menu that are shown (ignoring scrolling). In +# show-all mode, this list contains all items in _cur_menu. Otherwise, it +# contains just the visible items. +# +# _sel_node_i: +# Index in _shown of the currently selected node +# +# _menu_scroll: +# Index in _shown of the top row of the main display +# +# _parent_screen_rows: +# List/stack of the row numbers that the selections in the parent menus +# appeared on. This is used to prevent the scrolling from jumping around +# when going in and out of menus. +# +# _show_all: +# If True, "show-all" mode is on. Show-all mode shows all symbols and other +# items in the current menu, including those that lack a prompt or aren't +# currently visible. +# +# Invisible items are drawn in a different style to make them stand out. +# +# _show_name: +# If True, the names of all symbol are shown in addition to the prompt. +# +# _conf_changed: +# True if the configuration has been changed. If False, we don't bother +# showing the save-and-quit dialog. +# +# We reset this to False whenever the configuration is saved explicitly +# from the save dialog. + +def _menuconfig(stdscr): + # Logic for the main display, with the list of symbols, etc. + + globals()["stdscr"] = stdscr + global _conf_changed + global _show_name + + _init() + + while True: + _draw_main() + curses.doupdate() + + + c = _get_wch_compat(_menu_win) + + if c == curses.KEY_RESIZE: + _resize_main() + + if c in (curses.KEY_DOWN, "j", "J"): + _select_next_menu_entry() + + elif c in (curses.KEY_UP, "k", "K"): + _select_prev_menu_entry() + + elif c in (curses.KEY_NPAGE, "\x04"): # Page Down/Ctrl-D + # Keep it simple. This way we get sane behavior for small windows, + # etc., for free. + for _ in range(_PG_JUMP): + _select_next_menu_entry() + + elif c in (curses.KEY_PPAGE, "\x15"): # Page Up/Ctrl-U + for _ in range(_PG_JUMP): + _select_prev_menu_entry() + + elif c in (curses.KEY_END, "G"): + _select_last_menu_entry() + + elif c in (curses.KEY_HOME, "g"): + _select_first_menu_entry() + + elif c in (curses.KEY_RIGHT, " ", "\n", "l", "L"): + # Do appropriate node action. Only Space is treated specially, + # preferring to toggle nodes rather than enter menus. + + sel_node = _shown[_sel_node_i] + + if sel_node.is_menuconfig and not \ + (c == " " and _prefer_toggle(sel_node.item)): + + _enter_menu(sel_node) + + else: + _change_node(sel_node) + if _is_y_mode_choice_sym(sel_node.item): + # Immediately jump to the parent menu after making a choice + # selection, like 'make menuconfig' does + _leave_menu() + + elif c in ("n", "N"): + _set_sel_node_tri_val(0) + + elif c in ("m", "M"): + _set_sel_node_tri_val(1) + + elif c in ("y", "Y"): + _set_sel_node_tri_val(2) + + elif c in (curses.KEY_LEFT, curses.KEY_BACKSPACE, _ERASE_CHAR, + "\x1B", # \x1B = ESC + "h", "H"): + + if c == "\x1B" and _cur_menu is _kconf.top_node: + res = quit_dialog() + if res: + return res + else: + _leave_menu() + + elif c in ("o", "O"): + if _conf_changed: + c = _key_dialog( + "Load", + "You have unsaved changes. Load new\n" + "configuration anyway?\n" + "\n" + " (Y)es (C)ancel", + "yc") + + if c is None or c == "c": + continue + + if _load_dialog(): + _conf_changed = False + + elif c in ("s", "S"): + if _save_dialog(_kconf.write_config, _config_filename, + "configuration"): + + _conf_changed = False + + elif c in ("d", "D"): + _save_dialog(_kconf.write_min_config, "defconfig", + "minimal configuration") + + elif c == "/": + _jump_to_dialog() + # The terminal might have been resized while the fullscreen jump-to + # dialog was open + _resize_main() + + elif c == "?": + _info_dialog(_shown[_sel_node_i], False) + # The terminal might have been resized while the fullscreen info + # dialog was open + _resize_main() + + elif c in ("a", "A"): + _toggle_show_all() + + elif c in ("c", "C"): + _show_name = not _show_name + + elif c in ("q", "Q"): + res = quit_dialog() + if res: + return res + +def quit_dialog(): + if not _conf_changed: + return "No changes to save" + + while True: + c = _key_dialog( + "Quit", + " Save configuration?\n" + "\n" + "(Y)es (N)o (C)ancel", + "ync") + + if c is None or c == "c": + return None + + if c == "y": + if _try_save(_kconf.write_config, _config_filename, + "configuration"): + + return "Configuration saved to '{}'" \ + .format(_config_filename) + + elif c == "n": + return "Configuration was not saved" + +def _init(): + # Initializes the main display with the list of symbols, etc. Also does + # misc. global initialization that needs to happen after initializing + # curses. + + global _ERASE_CHAR + + global _path_win + global _top_sep_win + global _menu_win + global _bot_sep_win + global _help_win + + global _parent_screen_rows + global _cur_menu + global _shown + global _sel_node_i + global _menu_scroll + + global _show_name + + # Looking for this in addition to KEY_BACKSPACE (which is unreliable) makes + # backspace work with TERM=vt100. That makes it likely to work in sane + # environments. + # + # erasechar() returns a 'bytes' object. Since we use get_wch(), we need to + # decode it. Just give up and avoid crashing if it can't be decoded. + _ERASE_CHAR = curses.erasechar().decode("utf-8", "ignore") + + _init_styles() + + # Hide the cursor + _safe_curs_set(0) + + # Initialize windows + + # Top row, with menu path + _path_win = _styled_win(_PATH_STYLE) + + # Separator below menu path, with title and arrows pointing up + _top_sep_win = _styled_win(_SEPARATOR_STYLE) + + # List of menu entries with symbols, etc. + _menu_win = _styled_win(_LIST_STYLE) + _menu_win.keypad(True) + + # Row below menu list, with arrows pointing down + _bot_sep_win = _styled_win(_SEPARATOR_STYLE) + + # Help window with keys at the bottom + _help_win = _styled_win(_HELP_STYLE) + + # The rows we'd like the nodes in the parent menus to appear on. This + # prevents the scroll from jumping around when going in and out of menus. + _parent_screen_rows = [] + + # Initial state + + _cur_menu = _kconf.top_node + _shown = _shown_nodes(_cur_menu) + _sel_node_i = 0 + _menu_scroll = 0 + + _show_name = False + + # Give windows their initial size + _resize_main() + +def _resize_main(): + # Resizes the main display, with the list of symbols, etc., to fill the + # terminal + + global _menu_scroll + + screen_height, screen_width = stdscr.getmaxyx() + + _path_win.resize(1, screen_width) + _top_sep_win.resize(1, screen_width) + _bot_sep_win.resize(1, screen_width) + + help_win_height = len(_MAIN_HELP_LINES) + menu_win_height = screen_height - help_win_height - 3 + + if menu_win_height >= 1: + _menu_win.resize(menu_win_height, screen_width) + _help_win.resize(help_win_height, screen_width) + + _top_sep_win.mvwin(1, 0) + _menu_win.mvwin(2, 0) + _bot_sep_win.mvwin(2 + menu_win_height, 0) + _help_win.mvwin(2 + menu_win_height + 1, 0) + else: + # Degenerate case. Give up on nice rendering and just prevent errors. + + menu_win_height = 1 + + _menu_win.resize(1, screen_width) + _help_win.resize(1, screen_width) + + for win in _top_sep_win, _menu_win, _bot_sep_win, _help_win: + win.mvwin(0, 0) + + # Adjust the scroll so that the selected node is still within the window, + # if needed + if _sel_node_i - _menu_scroll >= menu_win_height: + _menu_scroll = _sel_node_i - menu_win_height + 1 + +def _menu_win_height(): + # Returns the height of the menu display + + return _menu_win.getmaxyx()[0] + +def _prefer_toggle(item): + # For nodes with menus, determines whether Space should change the value of + # the node's item or enter its menu. We toggle symbols (which have menus + # when they're defined with 'menuconfig') and choices that can be in more + # than one mode (e.g. optional choices). In other cases, we enter the menu. + + return isinstance(item, Symbol) or \ + (isinstance(item, Choice) and len(item.assignable) > 1) + +def _enter_menu(menu): + # Makes 'menu' the currently displayed menu + + global _cur_menu + global _shown + global _sel_node_i + global _menu_scroll + + shown_sub = _shown_nodes(menu) + # Never enter empty menus. We depend on having a current node. + if shown_sub: + # Remember where the current node appears on the screen, so we can try + # to get it to appear in the same place when we leave the menu + _parent_screen_rows.append(_sel_node_i - _menu_scroll) + + # Jump into menu + _cur_menu = menu + _shown = shown_sub + _sel_node_i = 0 + _menu_scroll = 0 + +def _jump_to(node): + # Jumps directly to the menu node 'node' + + global _cur_menu + global _shown + global _sel_node_i + global _menu_scroll + global _show_all + global _parent_screen_rows + + # Clear remembered menu locations. We might not even have been in the + # parent menus before. + _parent_screen_rows = [] + + # Turn on show-all mode if the node isn't visible + if not (node.prompt and expr_value(node.prompt[1])): + _show_all = True + + _cur_menu = _parent_menu(node) + _shown = _shown_nodes(_cur_menu) + _sel_node_i = _shown.index(node) + + # Center the jumped-to node vertically, if possible + _menu_scroll = max(_sel_node_i - _menu_win_height()//2, 0) + +def _leave_menu(): + # Jumps to the parent menu of the current menu. Does nothing if we're in + # the top menu. + + global _cur_menu + global _shown + global _sel_node_i + global _menu_scroll + + if _cur_menu is _kconf.top_node: + return + + # Jump to parent menu + parent = _parent_menu(_cur_menu) + _shown = _shown_nodes(parent) + _sel_node_i = _shown.index(_cur_menu) + _cur_menu = parent + + # Try to make the menu entry appear on the same row on the screen as it did + # before we entered the menu. + + if _parent_screen_rows: + # The terminal might have shrunk since we were last in the parent menu + screen_row = min(_parent_screen_rows.pop(), _menu_win_height() - 1) + _menu_scroll = max(_sel_node_i - screen_row, 0) + else: + # No saved parent menu locations, meaning we jumped directly to some + # node earlier. Just center the node vertically if possible. + _menu_scroll = max(_sel_node_i - _menu_win_height()//2, 0) + +def _select_next_menu_entry(): + # Selects the menu entry after the current one, adjusting the scroll if + # necessary. Does nothing if we're already at the last menu entry. + + global _sel_node_i + global _menu_scroll + + if _sel_node_i < len(_shown) - 1: + # Jump to the next node + _sel_node_i += 1 + + # If the new node is sufficiently close to the edge of the menu window + # (as determined by _SCROLL_OFFSET), increase the scroll by one. This + # gives nice and non-jumpy behavior even when + # _SCROLL_OFFSET >= _menu_win_height(). + if _sel_node_i >= _menu_scroll + _menu_win_height() - _SCROLL_OFFSET: + _menu_scroll = min(_menu_scroll + 1, + _max_scroll(_shown, _menu_win)) + +def _select_prev_menu_entry(): + # Selects the menu entry before the current one, adjusting the scroll if + # necessary. Does nothing if we're already at the first menu entry. + + global _sel_node_i + global _menu_scroll + + if _sel_node_i > 0: + # Jump to the previous node + _sel_node_i -= 1 + + # See _select_next_menu_entry() + if _sel_node_i <= _menu_scroll + _SCROLL_OFFSET: + _menu_scroll = max(_menu_scroll - 1, 0) + +def _select_last_menu_entry(): + # Selects the last menu entry in the current menu + + global _sel_node_i + global _menu_scroll + + _sel_node_i = len(_shown) - 1 + _menu_scroll = _max_scroll(_shown, _menu_win) + +def _select_first_menu_entry(): + # Selects the first menu entry in the current menu + + global _sel_node_i + global _menu_scroll + + _sel_node_i = _menu_scroll = 0 + +def _toggle_show_all(): + # Toggles show-all mode on/off + + global _show_all + global _shown + global _sel_node_i + global _menu_scroll + + # Row on the screen the cursor is on. Preferably we want the same row to + # stay highlighted. + old_row = _sel_node_i - _menu_scroll + + _show_all = not _show_all + # List of new nodes to be shown after toggling _show_all + new_shown = _shown_nodes(_cur_menu) + + # Find a good node to select. The selected node might disappear if show-all + # mode is turned off. + + # If there are visible nodes before the previously selected node, select + # the closest one. This will select the previously selected node itself if + # it is still visible. + for node in reversed(_shown[:_sel_node_i + 1]): + if node in new_shown: + _sel_node_i = new_shown.index(node) + break + else: + # No visible nodes before the previously selected node. Select the + # closest visible node after it instead. + for node in _shown[_sel_node_i + 1:]: + if node in new_shown: + _sel_node_i = new_shown.index(node) + break + else: + # No visible nodes at all, meaning show-all was turned off inside + # an invisible menu. Don't allow that, as the implementation relies + # on always having a selected node. + _show_all = True + + return + + _shown = new_shown + + # Try to make the cursor stay on the same row in the menu window. This + # might be impossible if too many nodes have disappeared above the node. + _menu_scroll = max(_sel_node_i - old_row, 0) + +def _draw_main(): + # Draws the "main" display, with the list of symbols, the header, and the + # footer. + # + # This could be optimized to only update the windows that have actually + # changed, but keep it simple for now and let curses sort it out. + + term_width = stdscr.getmaxyx()[1] + + + # + # Update the top row with the menu path + # + + _path_win.erase() + + # Draw the menu path ("(top menu) -> menu -> submenu -> ...") + + menu_prompts = [] + + menu = _cur_menu + while menu is not _kconf.top_node: + menu_prompts.append(menu.prompt[0]) + menu = _parent_menu(menu) + menu_prompts.append("(top menu)") + menu_prompts.reverse() + + # Hack: We can't put ACS_RARROW directly in the string. Temporarily + # represent it with NULL. Maybe using a Unicode character would be better. + menu_path_str = " \0 ".join(menu_prompts) + + # Scroll the menu path to the right if needed to make the current menu's + # title visible + if len(menu_path_str) > term_width: + menu_path_str = menu_path_str[len(menu_path_str) - term_width:] + + # Print the path with the arrows reinserted + split_path = menu_path_str.split("\0") + _safe_addstr(_path_win, split_path[0]) + for s in split_path[1:]: + _safe_addch(_path_win, curses.ACS_RARROW) + _safe_addstr(_path_win, s) + + _path_win.noutrefresh() + + + # + # Update the separator row below the menu path + # + + _top_sep_win.erase() + + # Draw arrows pointing up if the symbol window is scrolled down. Draw them + # before drawing the title, so the title ends up on top for small windows. + if _menu_scroll > 0: + _safe_hline(_top_sep_win, 0, 4, curses.ACS_UARROW, _N_SCROLL_ARROWS) + + # Add the 'mainmenu' text as the title, centered at the top + _safe_addstr(_top_sep_win, + 0, (term_width - len(_kconf.mainmenu_text))//2, + _kconf.mainmenu_text) + + _top_sep_win.noutrefresh() + + + # + # Update the symbol window + # + + _menu_win.erase() + + # Draw the _shown nodes starting from index _menu_scroll up to either as + # many as fit in the window, or to the end of _shown + for i in range(_menu_scroll, + min(_menu_scroll + _menu_win_height(), len(_shown))): + + node = _shown[i] + + if node.prompt and expr_value(node.prompt[1]): + style = _LIST_SEL_STYLE if i == _sel_node_i else _LIST_STYLE + else: + style = _LIST_INVISIBLE_SEL_STYLE if i == _sel_node_i else \ + _LIST_INVISIBLE_STYLE + + _safe_addstr(_menu_win, i - _menu_scroll, 0, _node_str(node), style) + + _menu_win.noutrefresh() + + + # + # Update the bottom separator window + # + + _bot_sep_win.erase() + + # Draw arrows pointing down if the symbol window is scrolled up + if _menu_scroll < _max_scroll(_shown, _menu_win): + _safe_hline(_bot_sep_win, 0, 4, curses.ACS_DARROW, _N_SCROLL_ARROWS) + + # Indicate when show-all and/or show-name mode is enabled + enabled_modes = [] + if _show_all: + enabled_modes.append("show-all") + if _show_name: + enabled_modes.append("show-name") + if enabled_modes: + s = " and ".join(enabled_modes) + " mode enabled" + _safe_addstr(_bot_sep_win, 0, term_width - len(s) - 2, s) + + _bot_sep_win.noutrefresh() + + + # + # Update the help window + # + + _help_win.erase() + + for i, line in enumerate(_MAIN_HELP_LINES): + _safe_addstr(_help_win, i, 0, line) + + _help_win.noutrefresh() + +def _parent_menu(node): + # Returns the menu node of the menu that contains 'node'. In addition to + # proper 'menu's, this might also be a 'menuconfig' symbol or a 'choice'. + # "Menu" here means a menu in the interface. + + menu = node.parent + while not menu.is_menuconfig: + menu = menu.parent + return menu + +def _shown_nodes(menu): + # Returns a list of the nodes in 'menu' (see _parent_menu()) that should be + # shown in the menu window + + res = [] + + def rec(node): + nonlocal res + + while node: + # Show the node if its prompt is visible. For menus, also check + # 'visible if'. In show-all mode, show everything. + if _show_all or \ + (node.prompt and expr_value(node.prompt[1]) and not \ + (node.item == MENU and not expr_value(node.visibility))): + + res.append(node) + + # If a node has children but doesn't have the is_menuconfig + # flag set, the children come from a submenu created implicitly + # from dependencies. Show those in this menu too. + if node.list and not node.is_menuconfig: + rec(node.list) + + node = node.next + + rec(menu.list) + return res + +def _change_node(node): + # Changes the value of the menu node 'node' if it is a symbol. Bools and + # tristates are toggled, while other symbol types pop up a text entry + # dialog. + + if not isinstance(node.item, (Symbol, Choice)): + return + + # This will hit for invisible symbols in show-all mode + if not (node.prompt and expr_value(node.prompt[1])): + return + + # sc = symbol/choice + sc = node.item + + if sc.type in (INT, HEX, STRING): + s = sc.str_value + + while True: + s = _input_dialog("Value for '{}' ({})".format( + node.prompt[0], TYPE_TO_STR[sc.type]), + s, _range_info(sc)) + + if s is None: + break + + if sc.type in (INT, HEX): + s = s.strip() + + # 'make menuconfig' does this too. Hex values not starting with + # '0x' are accepted when loading .config files though. + if sc.type == HEX and not s.startswith(("0x", "0X")): + s = "0x" + s + + if _check_validity(sc, s): + _set_val(sc, s) + break + + elif len(sc.assignable) == 1: + # Handles choice symbols for choices in y mode, which are a special + # case: .assignable can be (2,) while .tri_value is 0. + _set_val(sc, sc.assignable[0]) + + else: + # Set the symbol to the value after the current value in + # sc.assignable, with wrapping + val_index = sc.assignable.index(sc.tri_value) + _set_val(sc, sc.assignable[(val_index + 1) % len(sc.assignable)]) + +def _set_sel_node_tri_val(tri_val): + # Sets the value of the currently selected menu entry to 'tri_val', if that + # value can be assigned + + sc = _shown[_sel_node_i].item + if isinstance(sc, (Symbol, Choice)) and tri_val in sc.assignable: + _set_val(sc, tri_val) + +def _set_val(sc, val): + # Wrapper around Symbol/Choice.set_value() for updating the menu state and + # _conf_changed + + global _conf_changed + + # Use the string representation of tristate values. This makes the format + # consistent for all symbol types. + if val in TRI_TO_STR: + val = TRI_TO_STR[val] + + if val != sc.str_value: + sc.set_value(val) + _conf_changed = True + + # Changing the value of the symbol might have changed what items in the + # current menu are visible. Recalculate the state. + _update_menu() + +def _update_menu(): + # Updates the current menu after the value of a symbol or choice has been + # changed. Changing a value might change which items in the menu are + # visible. + # + # Tries to preserve the location of the cursor when items disappear above + # it. + + global _shown + global _sel_node_i + global _menu_scroll + + # Row on the screen the cursor was on + old_row = _sel_node_i - _menu_scroll + + sel_node = _shown[_sel_node_i] + + # New visible nodes + _shown = _shown_nodes(_cur_menu) + + # New index of selected node + _sel_node_i = _shown.index(sel_node) + + # Try to make the cursor stay on the same row in the menu window. This + # might be impossible if too many nodes have disappeared above the node. + _menu_scroll = max(_sel_node_i - old_row, 0) + +def _input_dialog(title, initial_text, info_text=None): + # Pops up a dialog that prompts the user for a string + # + # title: + # Title to display at the top of the dialog window's border + # + # initial_text: + # Initial text to prefill the input field with + # + # info_text: + # String to show next to the input field. If None, just the input field + # is shown. + + win = _styled_win(_DIALOG_BODY_STYLE) + win.keypad(True) + + info_lines = info_text.split("\n") if info_text else [] + + # Give the input dialog its initial size + _resize_input_dialog(win, title, info_lines) + + _safe_curs_set(2) + + # Input field text + s = initial_text + + # Cursor position + i = len(initial_text) + + def edit_width(): + return win.getmaxyx()[1] - 4 + + # Horizontal scroll offset + hscroll = max(i - edit_width() + 1, 0) + + while True: + # Draw the "main" display with the menu, etc., so that resizing still + # works properly. This is like a stack of windows, only hardcoded for + # now. + _draw_main() + _draw_input_dialog(win, title, info_lines, s, i, hscroll) + curses.doupdate() + + + c = _get_wch_compat(win) + + if c == "\n": + _safe_curs_set(0) + return s + + if c == "\x1B": # \x1B = ESC + _safe_curs_set(0) + return None + + + if c == curses.KEY_RESIZE: + # Resize the main display too. The dialog floats above it. + _resize_main() + _resize_input_dialog(win, title, info_lines) + + else: + s, i, hscroll = _edit_text(c, s, i, hscroll, edit_width()) + +def _resize_input_dialog(win, title, info_lines): + # Resizes the input dialog to a size appropriate for the terminal size + + screen_height, screen_width = stdscr.getmaxyx() + + win_height = 5 + if info_lines: + win_height += len(info_lines) + 1 + win_height = min(win_height, screen_height) + + win_width = max(_INPUT_DIALOG_MIN_WIDTH, + len(title) + 4, + *(len(line) + 4 for line in info_lines)) + win_width = min(win_width, screen_width) + + win.resize(win_height, win_width) + win.mvwin((screen_height - win_height)//2, + (screen_width - win_width)//2) + +def _draw_input_dialog(win, title, info_lines, s, i, hscroll): + edit_width = win.getmaxyx()[1] - 4 + + win.erase() + + _draw_frame(win, title) + + # Note: Perhaps having a separate window for the input field would be nicer + visible_s = s[hscroll:hscroll + edit_width] + _safe_addstr(win, 2, 2, visible_s + " "*(edit_width - len(visible_s)), + _INPUT_FIELD_STYLE) + + for linenr, line in enumerate(info_lines): + _safe_addstr(win, 4 + linenr, 2, line) + + _safe_move(win, 2, 2 + i - hscroll) + + win.noutrefresh() + +def _load_dialog(): + # Dialog for loading a new configuration + # + # Return value: + # True if a new configuration was loaded, and False if the user canceled + # the dialog + + global _show_all + + filename = "" + while True: + filename = _input_dialog("File to load", filename, _load_save_info()) + + if filename is None: + return False + + filename = os.path.expanduser(filename) + + if _try_load(filename): + sel_node = _shown[_sel_node_i] + + # Turn on show-all mode if the current node is (no longer) visible + if not (sel_node.prompt and expr_value(sel_node.prompt[1])): + _show_all = True + + _update_menu() + + # The message dialog indirectly updates the menu display, so _msg() + # must be called after the new state has been initialized + _msg("Success", "Loaded {}".format(filename)) + return True + +def _try_load(filename): + # Tries to load a configuration file. Pops up an error and returns False on + # failure. + # + # filename: + # Configuration file to load + + # Hack: strerror and errno are lost after we raise the custom IOError with + # troubleshooting help in Kconfig.load_config(). Adding them back to the + # exception loses the custom message. As a workaround, try opening the file + # separately first and report any errors. + try: + open(filename).close() + except OSError as e: + _error("Error loading {}\n\n{} (errno: {})" + .format(filename, e.strerror, errno.errorcode[e.errno])) + return False + + try: + _kconf.load_config(filename) + return True + except OSError as e: + _error("Error loading {}\n\nUnknown error".format(filename)) + return False + +def _save_dialog(save_fn, default_filename, description): + # Dialog for saving the current configuration + # + # save_fn: + # Function to call with 'filename' to save the file + # + # default_filename: + # Prefilled filename in the input field + # + # description: + # String describing the thing being saved + # + # Return value: + # True if the configuration was saved, and False if the user canceled the + # dialog + + filename = default_filename + while True: + filename = _input_dialog("Filename to save {} to".format(description), + filename, _load_save_info()) + + if filename is None: + return False + + filename = os.path.expanduser(filename) + + if _try_save(save_fn, filename, description): + _msg("Success", "{} saved to {}".format(description, filename)) + return True + +def _try_save(save_fn, filename, description): + # Tries to save a configuration file. Pops up an error and returns False on + # failure. + # + # save_fn: + # Function to call with 'filename' to save the file + # + # description: + # String describing the thing being saved + + try: + save_fn(filename) + return True + except OSError as e: + _error("Error saving {} to '{}'\n\n{} (errno: {})" + .format(description, e.filename, e.strerror, + errno.errorcode[e.errno])) + return False + +def _key_dialog(title, text, keys): + # Pops up a dialog that can be closed by pressing a key + # + # title: + # Title to display at the top of the dialog window's border + # + # text: + # Text to show in the dialog + # + # keys: + # List of keys that will close the dialog. Other keys (besides ESC) are + # ignored. The caller is responsible for providing a hint about which + # keys can be pressed in 'text'. + # + # Return value: + # The key that was pressed to close the dialog. Uppercase characters are + # converted to lowercase. ESC will always close the dialog, and returns + # None. + + win = _styled_win(_DIALOG_BODY_STYLE) + win.keypad(True) + + _resize_key_dialog(win, text) + + while True: + # See _input_dialog() + _draw_main() + _draw_key_dialog(win, title, text) + curses.doupdate() + + + c = _get_wch_compat(win) + + if c == "\x1B": # \x1B = ESC + return None + + + if c == curses.KEY_RESIZE: + # Resize the main display too. The dialog floats above it. + _resize_main() + _resize_key_dialog(win, text) + + elif isinstance(c, str): + c = c.lower() + if c in keys: + return c + +def _resize_key_dialog(win, text): + # Resizes the key dialog to a size appropriate for the terminal size + + screen_height, screen_width = stdscr.getmaxyx() + + lines = text.split("\n") + + win_height = min(len(lines) + 4, screen_height) + win_width = min(max(len(line) for line in lines) + 4, screen_width) + + win.resize(win_height, win_width) + win.mvwin((screen_height - win_height)//2, + (screen_width - win_width)//2) + +def _draw_key_dialog(win, title, text): + win.erase() + _draw_frame(win, title) + + for i, line in enumerate(text.split("\n")): + _safe_addstr(win, 2 + i, 2, line) + + win.noutrefresh() + +def _draw_frame(win, title): + # Draw a frame around the inner edges of 'win', with 'title' at the top + + win_height, win_width = win.getmaxyx() + + win.attron(_DIALOG_FRAME_STYLE) + + # Draw top/bottom edge + _safe_hline(win, 0, 0, " ", win_width) + _safe_hline(win, win_height - 1, 0, " ", win_width) + + # Draw left/right edge + _safe_vline(win, 0, 0, " ", win_height) + _safe_vline(win, 0, win_width - 1, " ", win_height) + + # Draw title + _safe_addstr(win, 0, (win_width - len(title))//2, title) + + win.attroff(_DIALOG_FRAME_STYLE) + +def _jump_to_dialog(): + # Implements the jump-to dialog, where symbols can be looked up via + # incremental search and jumped to. + # + # Returns True if the user jumped to a symbol, and False if the dialog was + # canceled. + + # Search text + s = "" + # Previous search text + prev_s = None + # Search text cursor position + s_i = 0 + # Horizontal scroll offset + hscroll = 0 + + # Index of selected row + sel_node_i = 0 + # Index in 'matches' of the top row of the list + scroll = 0 + + # Edit box at the top + edit_box = _styled_win(_INPUT_FIELD_STYLE) + edit_box.keypad(True) + + # List of matches + matches_win = _styled_win(_LIST_STYLE) + + # Bottom separator, with arrows pointing down + bot_sep_win = _styled_win(_SEPARATOR_STYLE) + + # Help window with instructions at the bottom + help_win = _styled_win(_HELP_STYLE) + + # Give windows their initial size + _resize_jump_to_dialog(edit_box, matches_win, bot_sep_win, help_win, + sel_node_i, scroll) + + _safe_curs_set(2) + + # TODO: Code duplication with _select_{next,prev}_menu_entry(). Can this be + # factored out in some nice way? + + def select_next_match(): + nonlocal sel_node_i + nonlocal scroll + + if sel_node_i < len(matches) - 1: + sel_node_i += 1 + + if sel_node_i >= scroll + matches_win.getmaxyx()[0] - _SCROLL_OFFSET: + scroll = min(scroll + 1, _max_scroll(matches, matches_win)) + + def select_prev_match(): + nonlocal sel_node_i + nonlocal scroll + + if sel_node_i > 0: + sel_node_i -= 1 + + if sel_node_i <= scroll + _SCROLL_OFFSET: + scroll = max(scroll - 1, 0) + + while True: + if s != prev_s: + # The search text changed. Find new matching nodes. + + prev_s = s + + try: + # We could use re.IGNORECASE here instead of lower(), but this + # is noticeably less jerky while inputting regexes like + # '.*debug$' (though the '.*' is redundant there). Those + # probably have bad interactions with re.search(), which + # matches anywhere in the string. + # + # It's not horrible either way. Just a bit smoother. + regex_searches = [re.compile(regex).search + for regex in s.lower().split()] + + # No exception thrown, so the regexes are okay + bad_re = None + + # List of matching nodes + matches = [] + + for node in _searched_nodes(): + for search in regex_searches: + # Does the regex match either the symbol name or the + # prompt (if any)? + if not (search(node.item.name.lower()) or + (node.prompt and + search(node.prompt[0].lower()))): + + # Give up on the first regex that doesn't match, to + # speed things up a bit when multiple regexes are + # entered + break + + else: + matches.append(node) + + except re.error as e: + # Bad regex. Remember the error message so we can show it. + bad_re = "Bad regular expression" + # re.error.msg was added in Python 3.5 + if hasattr(e, "msg"): + bad_re += ": " + e.msg + + matches = [] + + # Reset scroll and jump to the top of the list of matches + sel_node_i = scroll = 0 + + _draw_jump_to_dialog(edit_box, matches_win, bot_sep_win, help_win, + s, s_i, hscroll, + bad_re, matches, sel_node_i, scroll) + curses.doupdate() + + + c = _get_wch_compat(edit_box) + + if c == "\n": + if matches: + _jump_to(matches[sel_node_i]) + _safe_curs_set(0) + return True + + elif c == "\x1B": # \x1B = ESC + _safe_curs_set(0) + return False + + elif c == curses.KEY_RESIZE: + # We adjust the scroll so that the selected node stays visible in + # the list when the terminal is resized, hence the 'scroll' + # assignment + scroll = _resize_jump_to_dialog( + edit_box, matches_win, bot_sep_win, help_win, + sel_node_i, scroll) + + elif c == "\x06": # \x06 = Ctrl-F + _safe_curs_set(0) + _info_dialog(matches[sel_node_i], True) + _safe_curs_set(1) + + scroll = _resize_jump_to_dialog( + edit_box, matches_win, bot_sep_win, help_win, + sel_node_i, scroll) + + elif c == curses.KEY_DOWN: + select_next_match() + + elif c == curses.KEY_UP: + select_prev_match() + + elif c == curses.KEY_NPAGE: # Page Down + # Keep it simple. This way we get sane behavior for small windows, + # etc., for free. + for _ in range(_PG_JUMP): + select_next_match() + + elif c == curses.KEY_PPAGE: # Page Up + for _ in range(_PG_JUMP): + select_prev_match() + + else: + s, s_i, hscroll = _edit_text(c, s, s_i, hscroll, + edit_box.getmaxyx()[1] - 2) + +# Obscure Python: We never pass a value for cached_search_nodes, and it keeps +# pointing to the same list. This avoids a global. +def _searched_nodes(cached_search_nodes=[]): + # Returns a list of menu nodes to search, sorted by symbol name + + if not cached_search_nodes: + # Sort symbols by name and remove duplicates, then add all nodes for + # each symbol. + # + # Duplicates appear when symbols have multiple menu nodes (definition + # locations), but they appear in menu order, which isn't what we want + # here. We'd still need to go through sym.nodes as well. + for sym in sorted(set(_kconf.defined_syms), key=lambda sym: sym.name): + cached_search_nodes.extend(sym.nodes) + + return cached_search_nodes + +def _resize_jump_to_dialog(edit_box, matches_win, bot_sep_win, help_win, + sel_node_i, scroll): + # Resizes the jump-to dialog to fill the terminal. + # + # Returns the new scroll index. We adjust the scroll if needed so that the + # selected node stays visible. + + screen_height, screen_width = stdscr.getmaxyx() + + bot_sep_win.resize(1, screen_width) + + help_win_height = len(_JUMP_TO_HELP_LINES) + matches_win_height = screen_height - help_win_height - 4 + + if matches_win_height >= 1: + edit_box.resize(3, screen_width) + matches_win.resize(matches_win_height, screen_width) + help_win.resize(help_win_height, screen_width) + + matches_win.mvwin(3, 0) + bot_sep_win.mvwin(3 + matches_win_height, 0) + help_win.mvwin(3 + matches_win_height + 1, 0) + else: + # Degenerate case. Give up on nice rendering and just prevent errors. + + matches_win_height = 1 + + edit_box.resize(screen_height, screen_width) + matches_win.resize(1, screen_width) + help_win.resize(1, screen_width) + + for win in matches_win, bot_sep_win, help_win: + win.mvwin(0, 0) + + # Adjust the scroll so that the selected row is still within the window, if + # needed + if sel_node_i - scroll >= matches_win_height: + return sel_node_i - matches_win_height + 1 + return scroll + +def _draw_jump_to_dialog(edit_box, matches_win, bot_sep_win, help_win, + s, s_i, hscroll, + bad_re, matches, sel_node_i, scroll): + + edit_width = edit_box.getmaxyx()[1] - 2 + + + # + # Update list of matches + # + + matches_win.erase() + + if matches: + for i in range(scroll, + min(scroll + matches_win.getmaxyx()[0], len(matches))): + + sym = matches[i].item + + sym_str = '{}(="{}")'.format(sym.name, sym.str_value) + if matches[i].prompt: + sym_str += ' "{}"'.format(matches[i].prompt[0]) + + _safe_addstr(matches_win, i - scroll, 0, sym_str, + _LIST_SEL_STYLE if i == sel_node_i else _LIST_STYLE) + + else: + # bad_re holds the error message from the re.error exception on errors + _safe_addstr(matches_win, 0, 0, bad_re or "No matches") + + matches_win.noutrefresh() + + + # + # Update bottom separator line + # + + bot_sep_win.erase() + + # Draw arrows pointing down if the symbol list is scrolled up + if scroll < _max_scroll(matches, matches_win): + _safe_hline(bot_sep_win, 0, 4, curses.ACS_DARROW, _N_SCROLL_ARROWS) + + bot_sep_win.noutrefresh() + + + # + # Update help window at bottom + # + + help_win.erase() + + for i, line in enumerate(_JUMP_TO_HELP_LINES): + _safe_addstr(help_win, i, 0, line) + + help_win.noutrefresh() + + + # + # Update edit box. We do this last since it makes it handy to position the + # cursor. + # + + edit_box.erase() + + _draw_frame(edit_box, "Jump to symbol") + + # Draw arrows pointing up if the symbol list is scrolled down + if scroll > 0: + # TODO: Bit ugly that _DIALOG_FRAME_STYLE is repeated here + _safe_hline(edit_box, 2, 4, curses.ACS_UARROW, _N_SCROLL_ARROWS, + _DIALOG_FRAME_STYLE) + + # Note: Perhaps having a separate window for the input field would be nicer + visible_s = s[hscroll:hscroll + edit_width] + _safe_addstr(edit_box, 1, 1, visible_s, _INPUT_FIELD_STYLE) + + _safe_move(edit_box, 1, 1 + s_i - hscroll) + + edit_box.noutrefresh() + +def _info_dialog(node, from_jump_to_dialog): + # Shows a fullscreen window with information about 'node'. + # + # If 'from_jump_to_dialog' is True, the information dialog was opened from + # within the jump-to-dialog. In this case, we make '/' from within the + # information dialog just return, to avoid a confusing recursive invocation + # of the jump-to-dialog. + + # Top row, with title and arrows point up + top_line_win = _styled_win(_SEPARATOR_STYLE) + + # Text display + text_win = _styled_win(_INFO_TEXT_STYLE) + text_win.keypad(True) + + # Bottom separator, with arrows pointing down + bot_sep_win = _styled_win(_SEPARATOR_STYLE) + + # Help window with keys at the bottom + help_win = _styled_win(_HELP_STYLE) + + # Give windows their initial size + _resize_info_dialog(top_line_win, text_win, bot_sep_win, help_win) + + + # Get lines of help text + lines = _info_str(node).split("\n") + + # Index of first row in 'lines' to show + scroll = 0 + + while True: + _draw_info_dialog(node, lines, scroll, top_line_win, text_win, + bot_sep_win, help_win) + curses.doupdate() + + + c = _get_wch_compat(text_win) + + if c == curses.KEY_RESIZE: + _resize_info_dialog(top_line_win, text_win, bot_sep_win, help_win) + + elif c in (curses.KEY_DOWN, "j", "J"): + if scroll < _max_scroll(lines, text_win): + scroll += 1 + + elif c in (curses.KEY_NPAGE, "\x04"): # Page Down/Ctrl-D + scroll = min(scroll + _PG_JUMP, _max_scroll(lines, text_win)) + + elif c in (curses.KEY_PPAGE, "\x15"): # Page Up/Ctrl-U + scroll = max(scroll - _PG_JUMP, 0) + + elif c in (curses.KEY_END, "G"): + scroll = _max_scroll(lines, text_win) + + elif c in (curses.KEY_HOME, "g"): + scroll = 0 + + elif c in (curses.KEY_UP, "k", "K"): + if scroll > 0: + scroll -= 1 + + elif c == "/": + # Support starting a search from within the information dialog + + if from_jump_to_dialog: + # Avoid recursion + return + + if _jump_to_dialog(): + # Jumped to a symbol. Cancel the information dialog. + return + + # Stay in the information dialog if the jump-to dialog was + # canceled. Resize it in case the terminal was resized while the + # fullscreen jump-to dialog was open. + _resize_info_dialog(top_line_win, text_win, bot_sep_win, help_win) + + elif c in (curses.KEY_LEFT, curses.KEY_BACKSPACE, _ERASE_CHAR, + "\x1B", # \x1B = ESC + "q", "Q", "h", "H"): + + return + +def _resize_info_dialog(top_line_win, text_win, bot_sep_win, help_win): + # Resizes the info dialog to fill the terminal + + screen_height, screen_width = stdscr.getmaxyx() + + top_line_win.resize(1, screen_width) + bot_sep_win.resize(1, screen_width) + + help_win_height = len(_INFO_HELP_LINES) + text_win_height = screen_height - help_win_height - 2 + + if text_win_height >= 1: + text_win.resize(text_win_height, screen_width) + help_win.resize(help_win_height, screen_width) + + text_win.mvwin(1, 0) + bot_sep_win.mvwin(1 + text_win_height, 0) + help_win.mvwin(1 + text_win_height + 1, 0) + else: + # Degenerate case. Give up on nice rendering and just prevent errors. + + text_win.resize(1, screen_width) + help_win.resize(1, screen_width) + + for win in text_win, bot_sep_win, help_win: + win.mvwin(0, 0) + +def _draw_info_dialog(node, lines, scroll, top_line_win, text_win, + bot_sep_win, help_win): + + text_win_height, text_win_width = text_win.getmaxyx() + + + # + # Update top row + # + + top_line_win.erase() + + # Draw arrows pointing up if the information window is scrolled down. Draw + # them before drawing the title, so the title ends up on top for small + # windows. + if scroll > 0: + _safe_hline(top_line_win, 0, 4, curses.ACS_UARROW, _N_SCROLL_ARROWS) + + title = ("Symbol" if isinstance(node.item, Symbol) else + "Choice" if isinstance(node.item, Choice) else + "Menu" if node.item == MENU else + "Comment") + " information" + _safe_addstr(top_line_win, 0, (text_win_width - len(title))//2, title) + + top_line_win.noutrefresh() + + + # + # Update text display + # + + text_win.erase() + + for i, line in enumerate(lines[scroll:scroll + text_win_height]): + _safe_addstr(text_win, i, 0, line) + + text_win.noutrefresh() + + + # + # Update bottom separator line + # + + bot_sep_win.erase() + + # Draw arrows pointing down if the symbol window is scrolled up + if scroll < _max_scroll(lines, text_win): + _safe_hline(bot_sep_win, 0, 4, curses.ACS_DARROW, _N_SCROLL_ARROWS) + + bot_sep_win.noutrefresh() + + + # + # Update help window at bottom + # + + help_win.erase() + + for i, line in enumerate(_INFO_HELP_LINES): + _safe_addstr(help_win, i, 0, line) + + help_win.noutrefresh() + +def _info_str(node): + # Returns information about the menu node 'node' as a string. + # + # The helper functions are responsible for adding newlines. This allows + # them to return "" if they don't want to add any output. + + if isinstance(node.item, Symbol): + sym = node.item + + return ( + _name_info(sym) + + _prompt_info(sym) + + "Type: {}\n".format(TYPE_TO_STR[sym.type]) + + 'Value: "{}"\n\n'.format(sym.str_value) + + _help_info(sym) + + _direct_dep_info(sym) + + _defaults_info(sym) + + _select_imply_info(sym) + + _kconfig_def_info(sym) + ) + + if isinstance(node.item, Choice): + choice = node.item + + return ( + _name_info(choice) + + _prompt_info(choice) + + "Type: {}\n".format(TYPE_TO_STR[choice.type]) + + 'Mode: "{}"\n\n'.format(choice.str_value) + + _help_info(choice) + + _choice_syms_info(choice) + + _direct_dep_info(choice) + + _defaults_info(choice) + + _kconfig_def_info(choice) + ) + + # node.item in (MENU, COMMENT) + return _kconfig_def_info(node) + +def _name_info(sc): + # Returns a string with the name of the symbol/choice. Names are optional + # for choices. + + return "Name: {}\n".format(sc.name) if sc.name else "" + +def _prompt_info(sc): + # Returns a string listing the prompts of 'sc' (Symbol or Choice) + + s = "" + + for node in sc.nodes: + if node.prompt: + s += "Prompt: {}\n".format(node.prompt[0]) + + return s + +def _choice_syms_info(choice): + # Returns a string listing the choice symbols in 'choice'. Adds + # "(selected)" next to the selected one. + + s = "Choice symbols:\n" + + for sym in choice.syms: + s += " - " + sym.name + if sym is choice.selection: + s += " (selected)" + s += "\n" + + return s + "\n" + +def _help_info(sc): + # Returns a string with the help text(s) of 'sc' (Symbol or Choice). + # Symbols and choices defined in multiple locations can have multiple help + # texts. + + s = "" + + for node in sc.nodes: + if node.help is not None: + s += "Help:\n\n{}\n\n" \ + .format(textwrap.indent(node.help, " ")) + + return s + +def _direct_dep_info(sc): + # Returns a string describing the direct dependencies of 'sc' (Symbol or + # Choice). The direct dependencies are the OR of the dependencies from each + # definition location. The dependencies at each definition location come + # from 'depends on' and dependencies inherited from parent items. + + if sc.direct_dep is _kconf.y: + return "" + + return 'Direct dependencies (value: "{}"):\n{}\n' \ + .format(TRI_TO_STR[expr_value(sc.direct_dep)], + _split_expr_info(sc.direct_dep, 2)) + +def _defaults_info(sc): + # Returns a string describing the defaults of 'sc' (Symbol or Choice) + + if not sc.defaults: + return "" + + s = "Defaults:\n" + + for val, cond in sc.defaults: + s += " - " + if isinstance(sc, Symbol): + s += '{} (value: "{}")' \ + .format(_expr_str(val), TRI_TO_STR[expr_value(val)]) + else: + # Don't print the value next to the symbol name for choice + # defaults, as it looks a bit confusing + s += val.name + s += "\n" + + if cond is not _kconf.y: + s += ' Condition (value: "{}"):\n{}' \ + .format(TRI_TO_STR[expr_value(cond)], + _split_expr_info(cond, 7)) + + return s + "\n" + +def _split_expr_info(expr, indent): + # Returns a string with 'expr' split into its top-level && or || operands, + # with one operand per line, together with the operand's value. This is + # usually enough to get something readable for long expressions. A fancier + # recursive thingy would be possible too. + # + # indent: + # Number of leading spaces to add before the split expression. + + if len(split_expr(expr, AND)) > 1: + split_op = AND + op_str = "&&" + else: + split_op = OR + op_str = "||" + + s = "" + for i, term in enumerate(split_expr(expr, split_op)): + s += '{}{} {} (value: "{}")\n' \ + .format(" "*indent, + " " if i == 0 else op_str, + _expr_str(term), + TRI_TO_STR[expr_value(term)]) + return s + +def _select_imply_info(sym): + # Returns a string with information about which symbols 'select' or 'imply' + # 'sym'. The selecting/implying symbols are grouped according to which + # value they select/imply 'sym' to (n/m/y). + + s = "" + + def add_sis(expr, val, title): + nonlocal s + + # sis = selects/implies + sis = [si for si in split_expr(expr, OR) if expr_value(si) == val] + + if sis: + s += title + for si in sis: + s += " - {}\n".format(split_expr(si, AND)[0].name) + s += "\n" + + if sym.rev_dep is not _kconf.n: + add_sis(sym.rev_dep, 2, "Symbols currently y-selecting this symbol:\n") + add_sis(sym.rev_dep, 1, "Symbols currently m-selecting this symbol:\n") + add_sis(sym.rev_dep, 0, "Symbols currently n-selecting this symbol (no effect):\n") + + if sym.weak_rev_dep is not _kconf.n: + add_sis(sym.weak_rev_dep, 2, "Symbols currently y-implying this symbol:\n") + add_sis(sym.weak_rev_dep, 1, "Symbols currently m-implying this symbol:\n") + add_sis(sym.weak_rev_dep, 0, "Symbols currently n-implying this symbol (no effect):\n") + + return s + +def _kconfig_def_info(item): + # Returns a string with the definition of 'item' in Kconfig syntax, + # together with the definition location(s) + + nodes = [item] if isinstance(item, MenuNode) else item.nodes + + s = "Kconfig definition{}, with propagated dependencies\n" \ + .format("s" if len(nodes) > 1 else "") + s += (len(s) - 1)*"=" + "\n\n" + + s += "\n\n".join("At {}:{}, in menu {}:\n\n{}".format( + node.filename, node.linenr, _menu_path_info(node), + textwrap.indent(str(node), " ")) + for node in nodes) + + return s + +def _menu_path_info(node): + # Returns a string describing the menu path leading up to 'node' + + path = "" + + node = _parent_menu(node) + while node is not _kconf.top_node: + path = " -> " + node.prompt[0] + path + node = _parent_menu(node) + + return "(top menu)" + path + +def _styled_win(style): + # Returns a new curses window with background 'style' and space as the fill + # character. The initial dimensions are (1, 1), so the window needs to be + # sized and positioned separately. + + win = curses.newwin(1, 1) + win.bkgdset(" ", style) + return win + +def _max_scroll(lst, win): + # Assuming 'lst' is a list of items to be displayed in 'win', + # returns the maximum number of steps 'win' can be scrolled down. + # We stop scrolling when the bottom item is visible. + + return max(0, len(lst) - win.getmaxyx()[0]) + +def _edit_text(c, s, i, hscroll, width): + # Implements text editing commands for edit boxes. Takes a character (which + # could also be e.g. curses.KEY_LEFT) and the edit box state, and returns + # the new state after the character has been processed. + # + # c: + # Character from user + # + # s: + # Current contents of string + # + # i: + # Current cursor index in string + # + # hscroll: + # Index in s of the leftmost character in the edit box, for horizontal + # scrolling + # + # width: + # Width in characters of the edit box + # + # Return value: + # An (s, i, hscroll) tuple for the new state + + if c == curses.KEY_LEFT: + if i > 0: + i -= 1 + + elif c == curses.KEY_RIGHT: + if i < len(s): + i += 1 + + elif c in (curses.KEY_HOME, "\x01"): # \x01 = CTRL-A + i = 0 + + elif c in (curses.KEY_END, "\x05"): # \x05 = CTRL-E + i = len(s) + + elif c in (curses.KEY_BACKSPACE, _ERASE_CHAR): + if i > 0: + s = s[:i-1] + s[i:] + i -= 1 + + elif c == curses.KEY_DC: + s = s[:i] + s[i+1:] + + elif c == "\x17": # \x17 = CTRL-W + # The \W removes characters like ',' one at a time + new_i = re.search(r"(?:\w*|\W)\s*$", s[:i]).start() + s = s[:new_i] + s[i:] + i = new_i + + elif c == "\x0B": # \x0B = CTRL-K + s = s[:i] + + elif c == "\x15": # \x15 = CTRL-U + s = s[i:] + i = 0 + + elif isinstance(c, str): + # Insert character + + s = s[:i] + c + s[i:] + i += 1 + + # Adjust the horizontal scroll so that the cursor never touches the left or + # right edges of the edit box, except when it's at the beginning or the end + # of the string + if i < hscroll + _SCROLL_OFFSET: + hscroll = max(i - _SCROLL_OFFSET, 0) + elif i >= hscroll + width - _SCROLL_OFFSET: + max_scroll = max(len(s) - width + 1, 0) + hscroll = min(i - width + _SCROLL_OFFSET + 1, max_scroll) + + + return s, i, hscroll + +def _load_save_info(): + # Returns an information string for load/save dialog boxes + + return "(Relative to {})\n\nRefer to your home directory with ~" \ + .format(os.path.join(os.getcwd(), "")) + +def _msg(title, text): + # Pops up a message dialog that can be dismissed with Space/Enter/ESC + + _key_dialog(title, text, " \n") + +def _error(text): + # Pops up an error dialog that can be dismissed with Space/Enter/ESC + + _msg("Error", text) + +def _node_str(node): + # Returns the complete menu entry text for a menu node. + # + # Example return value: "[*] Support for X" + + # Calculate the indent to print the item with by checking how many levels + # above it the closest 'menuconfig' item is (this includes menus and + # choices as well as menuconfig symbols) + indent = 0 + parent = node.parent + while not parent.is_menuconfig: + indent += _SUBMENU_INDENT + parent = parent.parent + + # This approach gives nice alignment for empty string symbols ("() Foo") + s = "{:{}}".format(_value_str(node), 3 + indent) + + # 'not node.prompt' can only be True in show-all mode + if not node.prompt or \ + (_show_name and + (isinstance(node.item, Symbol) or + (isinstance(node.item, Choice) and node.item.name))): + + s += " <{}>".format(node.item.name) + + if node.prompt: + s += " " + if node.item == COMMENT: + s += "*** {} ***".format(node.prompt[0]) + else: + s += node.prompt[0] + + if isinstance(node.item, Symbol): + sym = node.item + + # Print "(NEW)" next to symbols without a user value (from e.g. a + # .config), but skip it for choice symbols in choices in y mode + if sym.user_value is None and \ + not (sym.choice and sym.choice.tri_value == 2): + + s += " (NEW)" + + if isinstance(node.item, Choice) and node.item.tri_value == 2: + # Print the prompt of the selected symbol after the choice for + # choices in y mode + sym = node.item.selection + if sym: + for node_ in sym.nodes: + if node_.prompt: + s += " ({})".format(node_.prompt[0]) + + # Print "--->" next to nodes that have menus that can potentially be + # entered. Add "(empty)" if the menu is empty. We don't allow those to be + # entered. + if node.is_menuconfig: + s += " --->" if _shown_nodes(node) else " ---> (empty)" + + return s + +def _value_str(node): + # Returns the value part ("[*]", "<M>", "(foo)" etc.) of a menu node + + item = node.item + + if item in (MENU, COMMENT): + return "" + + # Wouldn't normally happen, and generates a warning + if item.type == UNKNOWN: + return "" + + if item.type in (STRING, INT, HEX): + return "({})".format(item.str_value) + + # BOOL or TRISTATE + + if _is_y_mode_choice_sym(item): + return "(X)" if item.choice.selection is item else "( )" + + tri_val_str = (" ", "M", "*")[item.tri_value] + + if len(item.assignable) == 1: + # Pinned to a single value + return "" if isinstance(item, Choice) else "-{}-".format(tri_val_str) + + if item.type == BOOL: + return "[{}]".format(tri_val_str) + + # item.type == TRISTATE + if item.assignable == (1, 2): + return "{{{}}}".format(tri_val_str) # {M}/{*} + return "<{}>".format(tri_val_str) + +def _is_y_mode_choice_sym(item): + # The choice mode is an upper bound on the visibility of choice symbols, so + # we can check the choice symbols' own visibility to see if the choice is + # in y mode + return isinstance(item, Symbol) and item.choice and item.visibility == 2 + +def _check_validity(sym, s): + # Returns True if the string 's' is a well-formed value for 'sym'. + # Otherwise, displays an error and returns False. + + if sym.type not in (INT, HEX): + # Anything goes for non-int/hex symbols + return True + + base = 10 if sym.type == INT else 16 + + try: + int(s, base) + except ValueError: + _error("'{}' is a malformed {} value" + .format(s, TYPE_TO_STR[sym.type])) + return False + + for low_sym, high_sym, cond in sym.ranges: + if expr_value(cond): + low = int(low_sym.str_value, base) + val = int(s, base) + high = int(high_sym.str_value, base) + + if not low <= val <= high: + _error("{} is outside the range {}-{}" + .format(s, low_sym.str_value, high_sym.str_value)) + + return False + + break + + return True + +def _range_info(sym): + # Returns a string with information about the valid range for the symbol + # 'sym', or None if 'sym' isn't an int/hex symbol + + if sym.type not in (INT, HEX): + return None + + for low, high, cond in sym.ranges: + if expr_value(cond): + return "Range: {}-{}".format(low.str_value, high.str_value) + + return "No range constraints." + +def _is_num(name): + # Heuristic to see if a symbol name looks like a number, for nicer output + # when printing expressions. Things like 16 are actually symbol names, only + # they get their name as their value when the symbol is undefined. + + try: + int(name) + except ValueError: + if not name.startswith(("0x", "0X")): + return False + + try: + int(name, 16) + except ValueError: + return False + + return True + +def _get_wch_compat(win): + # Decent resizing behavior on PDCurses requires calling resize_term(0, 0) + # after receiving KEY_RESIZE, while NCURSES (usually) handles terminal + # resizing automatically in get(_w)ch() (see the end of the + # resizeterm(3NCURSES) man page). + # + # resize_term(0, 0) reliably fails and does nothing on NCURSES, so this + # hack gives NCURSES/PDCurses compatibility for resizing. I don't know + # whether it would cause trouble for other implementations. + + c = win.get_wch() + if c == curses.KEY_RESIZE: + try: + curses.resize_term(0, 0) + except curses.error: + pass + + return c + +# Ignore exceptions from some functions that might fail, e.g. for small +# windows. They usually do reasonable things anyway. + +def _safe_curs_set(visibility): + try: + curses.curs_set(visibility) + except curses.error: + pass + +def _safe_addstr(win, *args): + try: + win.addstr(*args) + except curses.error: + pass + +def _safe_addch(win, *args): + try: + win.addch(*args) + except curses.error: + pass + +def _safe_hline(win, *args): + try: + win.hline(*args) + except curses.error: + pass + +def _safe_vline(win, *args): + try: + win.vline(*args) + except curses.error: + pass + +def _safe_move(win, *args): + try: + win.move(*args) + except curses.error: + pass + +def _convert_c_lc_ctype_to_utf8(): + # See _CONVERT_C_LC_CTYPE_TO_UTF8 + + if _IS_WINDOWS: + # Windows rarely has issues here, and the PEP 538 implementation avoids + # changing the locale on it. None of the UTF-8 locales below were + # supported from some quick testing either. Play it safe. + return + + def _try_set_locale(loc): + try: + locale.setlocale(locale.LC_CTYPE, loc) + return True + except locale.Error: + return False + + # Is LC_CTYPE set to the C locale? + if locale.setlocale(locale.LC_CTYPE, None) == "C": + # This list was taken from the PEP 538 implementation in the CPython + # code, in Python/pylifecycle.c + for loc in "C.UTF-8", "C.utf8", "UTF-8": + if _try_set_locale(loc): + print("Note: Your environment is configured to use ASCII. To " + "avoid Unicode issues, LC_CTYPE was changed from the " + "C locale to the {} locale.".format(loc)) + break + +# Are we running on Windows? +_IS_WINDOWS = (platform.system() == "Windows") + +if __name__ == "__main__": + _main() diff --git a/scripts/read_config.py b/scripts/read_config.py new file mode 100644 index 0000000..de95af9 --- /dev/null +++ b/scripts/read_config.py @@ -0,0 +1,6 @@ +config = dict(map(lambda x: (x[0], int(x[1]) if x[1].isdecimal() else x[1][1:-1] if x[1].startswith('"') and x[1].endswith('"') else x[1]), map(lambda x: x.strip().split("=", 1), filter(lambda x: '=' in x, open(".config").readlines())))) + +if __name__ == "__main__": + import sys + if sys.argv[1] in config: + print(config[sys.argv[1]]) diff --git a/scripts/setup_wlan.sh b/scripts/setup_wlan.sh new file mode 100755 index 0000000..8621bf0 --- /dev/null +++ b/scripts/setup_wlan.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x +PORT=$1 +CONFIGFILE=$2 + +HOST=$(python3 ../scripts/read_config.py CONFIG_DHCP_HOSTNAME) +SSID=$(python3 ../scripts/read_config.py CONFIG_WPA_SSID) +PASS=$(python3 ../scripts/read_config.py CONFIG_WPA_KEYPHRASE) + +cat $PORT & +PRINTER_PID=$! + +# interrupt +echo -e -n "\003" >$PORT + +# Enter RAW Repl +echo -e -n "\001" >$PORT + +echo -e -n "import network\004" >$PORT +echo -e -n "wlan = network.WLAN(network.STA_IF)\004" >$PORT +echo -e -n "wlan.active(True)\004" >$PORT +echo -e -n "wlan.config(dhcp_hostname='$HOST')\004" >$PORT +echo -e -n "wlan.connect('$SSID', '$PASS')\004" >$PORT +echo -e -n "ap = network.WLAN(network.AP_IF)\004" >$PORT +echo -e -n "ap.active(False)\004" >$PORT +echo -e -n "\002" >$PORT + +sleep 3 + +kill $PRINTER_PID diff --git a/scripts/transfer.sh b/scripts/transfer.sh new file mode 100755 index 0000000..3e73b65 --- /dev/null +++ b/scripts/transfer.sh @@ -0,0 +1,46 @@ +#!/bin/bash +PORT=$1 +lname=$2 +rname=$3 + +cat $PORT & +PRINTER_PID=$! +trap "kill $PRINTER_PID; exit" SIGINT SIGTERM + +# interrupt +echo -e -n "\003" >$PORT +# Enter RAW Repl +echo -e -n "\001" >$PORT +sleep 0.5 + + +fsize=$(du -b $lname | cut -f 1) +echo "Size of $lname $fsize" +blocksize=200 +blocks=$(( $fsize / $blocksize )) +pos=0 +echo -e -n "__fh = open('$rname', 'wb')\n" >$PORT +echo -e -n "written = 0\n\004" >$PORT +sleep 0.1 + +echo -e -n "\003" >$PORT +while [[ $pos -lt $fsize ]]; do + encoded=$(dd if=$lname ibs=1 skip=$pos count=$blocksize 2>/dev/null | python3 -c 'import sys; print(repr(sys.stdin.buffer.read()), end="")') + echo -n "$pos..$(( $pos + $blocksize )) " + echo -n "written += __fh.write(${encoded})" > $PORT + echo -e -n "\n\004" > $PORT + pos=$(( $pos + $blocksize )) + sleep 0.08 +done +echo -e -n "__fh.close()\004" >$PORT +echo -e -n "del(__fh)\004" >$PORT +sleep 0.2 + +echo -e -n "print('\\\\n\\\\n%s bytes written to file $rname\\\\n' % written)\004" >$PORT + +# enter normal repl +echo -e -n "\002" >$PORT + +sleep 0.2 + +kill $PRINTER_PID diff --git a/stromzähler3/Kconfig b/stromzähler3/Kconfig new file mode 100644 index 0000000..afb1c7a --- /dev/null +++ b/stromzähler3/Kconfig @@ -0,0 +1,16 @@ +mainmenu "configuration" + +source ../scripts/Kconfig-wifi + +menu "Influxdb" + +config INFLUXDB_HOST + string "Hostname of influxdb server" + +config INFLUXDB_PORT + int "Port number" + +config INFLUXDB_PATH + string "Path element for POST request" + +endmenu diff --git a/stromzähler3/boot.py b/stromzähler3/boot.py new file mode 100644 index 0000000..0d30893 --- /dev/null +++ b/stromzähler3/boot.py @@ -0,0 +1,2 @@ +config = dict(map(lambda x: (x[0], int(x[1]) if x[1].isdigit() else x[1][1:-1] if x[1].startswith('"') and x[1].endswith('"') else x[1]), map(lambda x: x.strip().split("=", 1), filter(lambda x: '=' in x, open(".config").readlines())))) + diff --git a/stromzähler3/main.py b/stromzähler3/main.py new file mode 100644 index 0000000..65499ff --- /dev/null +++ b/stromzähler3/main.py @@ -0,0 +1,105 @@ +#!micropython +print("Start main.py") + +import os +import machine +import socket +import time +import micropython as mp + + +HOST = config['CONFIG_INFLUXDB_HOST'] +PORT = config['CONFIG_INFLUXDB_PORT'] +PATH = config['CONFIG_INFLUXDB_PATH'] + +H_REQUEST = bytes('POST /%s HTTP/1.1\r\n' % PATH, 'utf8') +H_HOST = bytes('Host: %s\r\n' % HOST, 'utf8') +H_CONTENT_TYPE = bytes('Content-Type: application/x-www-urlencoded\r\n', 'utf8') +H_CONTENT_LENGTH = bytes('Content-Length: %s\r\n', 'utf8') +H_CONNECTION = bytes('Connection: close\r\n', 'utf8') +H_NL = bytes('\r\n', 'utf8') + +def post(data): + print('[%d] Send HTTP POST: %s. Response: ' % (time.ticks_ms(), data), end='') + addr = socket.getaddrinfo(HOST, PORT)[0][-1] + data = bytes(data, 'utf8') + s = socket.socket() + s.connect(addr) + s.send(H_REQUEST) + s.send(H_HOST) + s.send(H_CONTENT_TYPE) + s.send(H_CONTENT_LENGTH % len(data)) + s.send(H_CONNECTION) + s.send(H_NL) + s.send(data) + + first_data = s.recv(100) + if first_data: + line, *_ = first_data.split(b'\r\n') + print(line, end='') + while data: + data = s.recv(100) + s.close() + print('') + +d4_total = 0 +d4_count = 0 + +d1_total = 0 +d1_count = 0 + +def update(t): + global d4_total, d4_count, d1_total, d1_count + d4 = d4_count + d1 = d1_count + post('stromzaehler3 d4_count=%d,d4_total=%d,d1_count=%d,d1_total=%d' % ( + d4, d4_total, d1, d1_total)) + d4_count -= d4 + d1_count -= d1 + +timer1 = machine.Timer(-1) +timer1.init(period=10000, mode=machine.Timer.PERIODIC, callback=update) + +def persist(t): + write_persistent_int("d4_total.txt", d4_total) + write_persistent_int("d1_total.txt", d1_total) + + +# Import +def cb_d4(p): + global d4_total, d4_count + v = p.value() + if v != 1: # skip rising edge + d4_total += 1 + d4_count += 1 + +d4 = machine.Pin(2, machine.Pin.IN, pull=machine.Pin.PULL_UP) +d4.irq(handler=lambda p: mp.schedule(cb_d4, p)) + +# Export +def cb_d1(p): + global d1_total, d1_count + v = p.value() + if v != 1: # skip rising edge + d1_count += 1 + d1_total += 1 + +d1 = machine.Pin(5, machine.Pin.IN) +d1.irq(handler=lambda p: mp.schedule(cb_d1, p)) + +# Pumpe +d3_last_value = 0 + +def cb_d3(p): + global d3_last_value + print('[%d] %s = %d' % (time.ticks_ms(), repr(p), p.value())) + v = 1 - p.value() + if v != d3_last_value: + post('stromzaehler3.Pumpe Status=%s' % v) + d3_last_value = v + +d3 = machine.Pin(0, machine.Pin.IN, pull=machine.Pin.PULL_UP) +d3.irq(handler=lambda p: mp.schedule(cb_d3, p)) + +print("main.py finished") + diff --git a/stromzähler3/make.sh b/stromzähler3/make.sh new file mode 100755 index 0000000..a76ff1a --- /dev/null +++ b/stromzähler3/make.sh @@ -0,0 +1,5 @@ +#!/bin/sh +PY_SOURCES="boot.py main.py .config" + +. ${0%/*}/../scripts/make.inc.sh + diff --git a/temp0/Kconfig b/temp0/Kconfig new file mode 100644 index 0000000..afb1c7a --- /dev/null +++ b/temp0/Kconfig @@ -0,0 +1,16 @@ +mainmenu "configuration" + +source ../scripts/Kconfig-wifi + +menu "Influxdb" + +config INFLUXDB_HOST + string "Hostname of influxdb server" + +config INFLUXDB_PORT + int "Port number" + +config INFLUXDB_PATH + string "Path element for POST request" + +endmenu diff --git a/temp0/boot.py b/temp0/boot.py new file mode 100644 index 0000000..0d30893 --- /dev/null +++ b/temp0/boot.py @@ -0,0 +1,2 @@ +config = dict(map(lambda x: (x[0], int(x[1]) if x[1].isdigit() else x[1][1:-1] if x[1].startswith('"') and x[1].endswith('"') else x[1]), map(lambda x: x.strip().split("=", 1), filter(lambda x: '=' in x, open(".config").readlines())))) + diff --git a/temp0/main.py b/temp0/main.py new file mode 100644 index 0000000..ac1de7d --- /dev/null +++ b/temp0/main.py @@ -0,0 +1,121 @@ +#!micropython +import time +print("[{}] Start main.py".format(time.ticks_ms())) + +import os +import gc +import socket +import machine +import onewire +import ds18x20 +import network + +HOST = config['CONFIG_INFLUXDB_HOST'] +PORT = config['CONFIG_INFLUXDB_PORT'] +PATH = config['CONFIG_INFLUXDB_PATH'] + +H_REQUEST = bytes('POST /%s HTTP/1.1\r\n' % PATH, 'utf8') +H_HOST = bytes('Host: %s\r\n' % HOST, 'utf8') +H_CONTENT_TYPE = bytes('Content-Type: application/x-www-urlencoded\r\n', 'utf8') +H_CONTENT_LENGTH = bytes('Content-Length: %s\r\n', 'utf8') +H_CONNECTION = bytes('Connection: close\r\n', 'utf8') +H_NL = bytes('\r\n', 'utf8') + +def post(data): + print('[%d] Send HTTP POST: %s. Response: ' % (time.ticks_ms(), data), end='') + addr = socket.getaddrinfo(HOST, PORT)[0][-1] + data = bytes(data, 'utf8') + s = socket.socket() + s.connect(addr) + s.send(H_REQUEST) + s.send(H_HOST) + s.send(H_CONTENT_TYPE) + s.send(H_CONTENT_LENGTH % len(data)) + s.send(H_CONNECTION) + s.send(H_NL) + s.send(data) + + first_data = s.recv(100) + if first_data: + line, *_ = first_data.split(b'\r\n') + print(line, end='') + while data: + data = s.recv(100) + s.close() + print('') + + + +pins = [5,4,0,2,14] +ow_list = list(map(onewire.OneWire, map(machine.Pin, pins))) +ds_list = list(map(ds18x20.DS18X20, ow_list)) + + +def onewire_name(ba): + return "-".join(map(lambda x: "{:02X}".format(x), ba)) + + +def update(func): + global ds_list + data = [] + + for i in range(len(pins)): + pin_id = pins[i] + print("[{}] Read pin {}".format(time.ticks_ms(), pin_id)) + ds = ds_list[i] + + roms = ds.scan() + if len(roms) > 0: + time.sleep_ms(500) + ds.convert_temp() + time.sleep_ms(1000) #750 + + for rom in roms: + value = ds.read_temp(rom) + data.append("temp0,pin={},sensor={} value={}".format(pin_id, onewire_name(rom), value)) + time.sleep_ms(100) + + data.append("temp0.ticks,pin={} value={}".format(pin_id, time.ticks_ms())) + func("\n".join(data)) + data = [] + + +wlan = network.WLAN(network.STA_IF) + + +def activate_wlan(): + global wlan + print("[{}] Connect to WLAN".format(time.ticks_ms())) + wlan.active(True) + wlan.connect() + while not wlan.isconnected(): + print("[{}] Connecting to WLAN".format(time.ticks_ms())) + time.sleep_ms(500) + print("[{}] Connected to WLAN".format(time.ticks_ms())) + +##### + +activate_wlan() + +print("[{}] Connected. Run loop".format(time.ticks_ms())) + +try: + while True: + gc.collect() + gc.disable() + print("[{}] Run update".format(time.ticks_ms())) + update(post) + gc.enable() + time.sleep_ms(2000) +except KeyboardInterrupt: + print("[{}] Loop aborted".format(time.ticks_ms())) +except Exception as e: + msg = repr(e) + print("[{}] {}".format(time.ticks_ms(), msg)) + try: + msg_esc = msg.replace("\"", "").replace("'", "").replace("\\", "") + post("temp0.exception value=\"{}\"".format(msg_esc)) + finally: + time.sleep_ms(1000) + machine.reset() + diff --git a/temp0/make.sh b/temp0/make.sh new file mode 100755 index 0000000..8266585 --- /dev/null +++ b/temp0/make.sh @@ -0,0 +1,5 @@ +#!/bin/sh +PY_SOURCES="main.py boot.py .config" + +. ${0%/*}/../scripts/make.inc.sh + |
