From 7c5dba8b77571aa6a2c0c38eeed7963b401d1cbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcelina=20Ko=C5=9Bcielnicka?= <mwk@0x04.net>
Date: Sun, 6 Feb 2022 10:10:21 +0100
Subject: Add memory_libmap pass.
---
passes/memory/memory_libmap.cc | 2093 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 2093 insertions(+)
create mode 100644 passes/memory/memory_libmap.cc
(limited to 'passes/memory/memory_libmap.cc')
diff --git a/passes/memory/memory_libmap.cc b/passes/memory/memory_libmap.cc
new file mode 100644
index 000000000..ab7bb7bb2
--- /dev/null
+++ b/passes/memory/memory_libmap.cc
@@ -0,0 +1,2093 @@
+/*
+ * yosys -- Yosys Open SYnthesis Suite
+ *
+ * Copyright (C) 2021 Marcelina Kościelnicka <mwk@0x04.net>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ */
+
+#include "memlib.h"
+
+#include <ctype.h>
+
+#include "kernel/yosys.h"
+#include "kernel/sigtools.h"
+#include "kernel/mem.h"
+#include "kernel/qcsat.h"
+
+USING_YOSYS_NAMESPACE
+PRIVATE_NAMESPACE_BEGIN
+
+using namespace MemLibrary;
+
+#define FACTOR_MUX 0.5
+#define FACTOR_DEMUX 0.5
+#define FACTOR_EMU 2
+
+struct PassOptions {
+ bool no_auto_distributed;
+ bool no_auto_block;
+ bool no_auto_huge;
+ double logic_cost_rom;
+ double logic_cost_ram;
+};
+
+struct WrPortConfig {
+ // Index of the read port this port is merged with, or -1 if none.
+ int rd_port;
+ // Index of the PortGroup in the Ram.
+ int port_group;
+ int port_variant;
+ const PortVariant *def;
+ // Emulate priority logic for this list of (source) write port indices.
+ std::vector<int> emu_prio;
+ // If true, this port needs to end up with uniform byte enables to work correctly.
+ bool force_uniform;
+
+ WrPortConfig() : rd_port(-1), force_uniform(false) {}
+};
+
+struct RdPortConfig {
+ // Index of the write port this port is merged with, or -1 if none.
+ int wr_port;
+ // Index of the PortGroup in the Ram.
+ int port_group;
+ int port_variant;
+ const PortVariant *def;
+ // If true, this is a sync port mapped into async mem, make an output
+ // register. Mutually exclusive with the following options.
+ bool emu_sync;
+ // Emulate the EN / ARST / SRST / init value circuitry.
+ bool emu_en;
+ bool emu_arst;
+ bool emu_srst;
+ bool emu_init;
+ // Emulate EN-SRST priority.
+ bool emu_srst_en_prio;
+ // If true, use clk_en as rd_en.
+ bool rd_en_to_clk_en;
+ // Emulate transparency logic for this list of (source) write port indices.
+ std::vector<int> emu_trans;
+
+ RdPortConfig() : wr_port(-1), emu_sync(false), emu_en(false), emu_arst(false), emu_srst(false), emu_init(false), emu_srst_en_prio(false), rd_en_to_clk_en(false) {}
+};
+
+// The named clock and clock polarity assignments.
+struct SharedClockConfig {
+ bool used;
+ SigBit clk;
+ // For anyedge clocks.
+ bool polarity;
+ // For non-anyedge clocks.
+ bool invert;
+};
+
+struct MemConfig {
+ // Reference to the library ram definition
+ const Ram *def;
+ // Port assignments, indexed by Mem port index.
+ std::vector<WrPortConfig> wr_ports;
+ std::vector<RdPortConfig> rd_ports;
+ std::vector<SharedClockConfig> shared_clocks;
+ // Emulate read-first write-read behavior using soft logic.
+ bool emu_read_first;
+ // This many low bits of (target) address are always-0 on all ports.
+ int base_width_log2;
+ int unit_width_log2;
+ std::vector<int> swizzle;
+ int hard_wide_mask;
+ int emu_wide_mask;
+ // How many times the base memory block will need to be duplicated to get more
+ // data bits.
+ int repl_d;
+ // How many times the whole memory array will need to be duplicated to cover
+ // all read ports required.
+ int repl_port;
+ // Emulation score — how much circuitry we need to add for priority / transparency /
+ // reset / initial value emulation.
+ int score_emu;
+ // Mux score — how much circuitry we need to add to manually decode whatever address
+ // bits are not decoded by the memory array itself, for reads.
+ int score_mux;
+ // Demux score — how much circuitry we need to add to manually decode whatever address
+ // bits are not decoded by the memory array itself, for writes.
+ int score_demux;
+ double cost;
+ MemConfig() : emu_read_first(false) {}
+};
+
+typedef std::vector<MemConfig> MemConfigs;
+
+struct MapWorker {
+ Module *module;
+ ModWalker modwalker;
+ SigMap sigmap;
+ SigMap sigmap_xmux;
+ FfInitVals initvals;
+
+ MapWorker(Module *module) : module(module), modwalker(module->design, module), sigmap(module), sigmap_xmux(module), initvals(&sigmap, module) {
+ for (auto cell : module->cells())
+ {
+ if (cell->type == ID($mux))
+ {
+ RTLIL::SigSpec sig_a = sigmap_xmux(cell->getPort(ID::A));
+ RTLIL::SigSpec sig_b = sigmap_xmux(cell->getPort(ID::B));
+
+ if (sig_a.is_fully_undef())
+ sigmap_xmux.add(cell->getPort(ID::Y), sig_b);
+ else if (sig_b.is_fully_undef())
+ sigmap_xmux.add(cell->getPort(ID::Y), sig_a);
+ }
+ }
+ }
+};
+
+struct SwizzleBit {
+ bool valid;
+ int mux_idx;
+ int addr;
+ int bit;
+};
+
+struct Swizzle {
+ int addr_shift;
+ int addr_start;
+ int addr_end;
+ std::vector<int> addr_mux_bits;
+ std::vector<std::vector<SwizzleBit>> bits;
+};
+
+struct MemMapping {
+ MapWorker &worker;
+ QuickConeSat qcsat;
+ Mem &mem;
+ const Library &lib;
+ const PassOptions &opts;
+ std::vector<MemConfig> cfgs;
+ bool logic_ok;
+ double logic_cost;
+ RamKind kind;
+ std::string style;
+ dict<int, int> wr_en_cache;
+ dict<std::pair<int, int>, bool> wr_implies_rd_cache;
+ dict<std::pair<int, int>, bool> wr_excludes_rd_cache;
+ dict<std::pair<int, int>, bool> wr_excludes_srst_cache;
+
+ MemMapping(MapWorker &worker, Mem &mem, const Library &lib, const PassOptions &opts) : worker(worker), qcsat(worker.modwalker), mem(mem), lib(lib), opts(opts) {
+ determine_style();
+ logic_ok = determine_logic_ok();
+ if (GetSize(mem.wr_ports) == 0)
+ logic_cost = mem.width * mem.size * opts.logic_cost_rom;
+ else
+ logic_cost = mem.width * mem.size * opts.logic_cost_ram;
+ if (kind == RamKind::Logic)
+ return;
+ for (int i = 0; i < GetSize(lib.rams); i++) {
+ auto &rdef = lib.rams[i];
+ if (!check_ram_kind(rdef))
+ continue;
+ if (!check_ram_style(rdef))
+ continue;
+ if (!check_init(rdef))
+ continue;
+ if (rdef.prune_rom && mem.wr_ports.empty())
+ continue;
+ MemConfig cfg;
+ cfg.def = &rdef;
+ for (auto &cdef: rdef.shared_clocks) {
+ (void)cdef;
+ SharedClockConfig clk;
+ clk.used = false;
+ cfg.shared_clocks.push_back(clk);
+ }
+ cfgs.push_back(cfg);
+ }
+ assign_wr_ports();
+ assign_rd_ports();
+ handle_trans();
+ // If we got this far, the memory is mappable. The following two can require emulating
+ // some functionality, but cannot cause the mapping to fail.
+ handle_priority();
+ handle_rd_rst();
+ score_emu_ports();
+ // Now it is just a matter of picking geometry.
+ handle_geom();
+ dump_configs(0);
+ prune_post_geom();
+ dump_configs(1);
+ }
+
+ bool addr_compatible(int wpidx, int rpidx) {
+ auto &wport = mem.wr_ports[wpidx];
+ auto &rport = mem.rd_ports[rpidx];
+ int max_wide_log2 = std::max(rport.wide_log2, wport.wide_log2);
+ SigSpec raddr = rport.addr.extract_end(max_wide_log2);
+ SigSpec waddr = wport.addr.extract_end(max_wide_log2);
+ int abits = std::max(GetSize(raddr), GetSize(waddr));
+ raddr.extend_u0(abits);
+ waddr.extend_u0(abits);
+ return worker.sigmap_xmux(raddr) == worker.sigmap_xmux(waddr);
+ }
+
+ int get_wr_en(int wpidx) {
+ auto it = wr_en_cache.find(wpidx);
+ if (it != wr_en_cache.end())
+ return it->second;
+ int res = qcsat.ez->expression(qcsat.ez->OpOr, qcsat.importSig(mem.wr_ports[wpidx].en));
+ wr_en_cache.insert({wpidx, res});
+ return res;
+ }
+
+ bool get_wr_implies_rd(int wpidx, int rpidx) {
+ auto key = std::make_pair(wpidx, rpidx);
+ auto it = wr_implies_rd_cache.find(key);
+ if (it != wr_implies_rd_cache.end())
+ return it->second;
+ int wr_en = get_wr_en(wpidx);
+ int rd_en = qcsat.importSigBit(mem.rd_ports[rpidx].en[0]);
+ qcsat.prepare();
+ bool res = !qcsat.ez->solve(wr_en, qcsat.ez->NOT(rd_en));
+ wr_implies_rd_cache.insert({key, res});
+ return res;
+ }
+
+ bool get_wr_excludes_rd(int wpidx, int rpidx) {
+ auto key = std::make_pair(wpidx, rpidx);
+ auto it = wr_excludes_rd_cache.find(key);
+ if (it != wr_excludes_rd_cache.end())
+ return it->second;
+ int wr_en = get_wr_en(wpidx);
+ int rd_en = qcsat.importSigBit(mem.rd_ports[rpidx].en[0]);
+ qcsat.prepare();
+ bool res = !qcsat.ez->solve(wr_en, rd_en);
+ wr_excludes_rd_cache.insert({key, res});
+ return res;
+ }
+
+ bool get_wr_excludes_srst(int wpidx, int rpidx) {
+ auto key = std::make_pair(wpidx, rpidx);
+ auto it = wr_excludes_srst_cache.find(key);
+ if (it != wr_excludes_srst_cache.end())
+ return it->second;
+ int wr_en = get_wr_en(wpidx);
+ int srst = qcsat.importSigBit(mem.rd_ports[rpidx].srst);
+ if (mem.rd_ports[rpidx].ce_over_srst) {
+ int rd_en = qcsat.importSigBit(mem.rd_ports[rpidx].en[0]);
+ srst = qcsat.ez->AND(srst, rd_en);
+ }
+ qcsat.prepare();
+ bool res = !qcsat.ez->solve(wr_en, srst);
+ wr_excludes_srst_cache.insert({key, res});
+ return res;
+ }
+
+ void dump_configs(int stage);
+ void dump_config(MemConfig &cfg);
+ void determine_style();
+ bool determine_logic_ok();
+ bool check_ram_kind(const Ram &ram);
+ bool check_ram_style(const Ram &ram);
+ bool check_init(const Ram &ram);
+ void assign_wr_ports();
+ void assign_rd_ports();
+ void handle_trans();
+ void handle_priority();
+ void handle_rd_rst();
+ void score_emu_ports();
+ void handle_geom();
+ void prune_post_geom();
+ void emit_port(const MemConfig &cfg, std::vector<Cell*> &cells, const PortVariant &pdef, const char *name, int wpidx, int rpidx, const std::vector<int> &hw_addr_swizzle);
+ void emit(const MemConfig &cfg);
+};
+
+void MemMapping::dump_configs(int stage) {
+ const char *stage_name;
+ switch (stage) {
+ case 0:
+ stage_name = "post-geometry";
+ break;
+ case 1:
+ stage_name = "after post-geometry prune";
+ break;
+ default:
+ abort();
+ }
+ log_debug("Memory %s.%s mapping candidates (%s):\n", log_id(mem.module->name), log_id(mem.memid), stage_name);
+ if (logic_ok) {
+ log_debug("- logic fallback\n");
+ log_debug(" - cost: %f\n", logic_cost);
+ }
+ for (auto &cfg: cfgs) {
+ dump_config(cfg);
+ }
+}
+
+void MemMapping::dump_config(MemConfig &cfg) {
+ log_debug("- %s:\n", log_id(cfg.def->id));
+ for (auto &it: cfg.def->options)
+ log_debug(" - option %s %s\n", it.first.c_str(), log_const(it.second));
+ log_debug(" - emulation score: %d\n", cfg.score_emu);
+ log_debug(" - replicates (for ports): %d\n", cfg.repl_port);
+ log_debug(" - replicates (for data): %d\n", cfg.repl_d);
+ log_debug(" - mux score: %d\n", cfg.score_mux);
+ log_debug(" - demux score: %d\n", cfg.score_demux);
+ log_debug(" - cost: %f\n", cfg.cost);
+ std::stringstream os;
+ for (int x: cfg.def->dbits)
+ os << " " << x;
+ std::string dbits_s = os.str();
+ log_debug(" - abits %d dbits%s\n", cfg.def->abits, dbits_s.c_str());
+ if (cfg.def->byte != 0)
+ log_debug(" - byte width %d\n", cfg.def->byte);
+ log_debug(" - chosen base width %d\n", cfg.def->dbits[cfg.base_width_log2]);
+ os.str("");
+ for (int x: cfg.swizzle)
+ if (x == -1)
+ os << " -";
+ else
+ os << " " << x;
+ std::string swizzle_s = os.str();
+ log_debug(" - swizzle%s\n", swizzle_s.c_str());
+ os.str("");
+ for (int i = 0; (1 << i) <= cfg.hard_wide_mask; i++)
+ if (cfg.hard_wide_mask & 1 << i)
+ os << " " << i;
+ std::string wide_s = os.str();
+ if (cfg.hard_wide_mask)
+ log_debug(" - hard wide bits%s\n", wide_s.c_str());
+ if (cfg.emu_read_first)
+ log_debug(" - emulate read-first behavior\n");
+ for (int i = 0; i < GetSize(mem.wr_ports); i++) {
+ auto &pcfg = cfg.wr_ports[i];
+ if (pcfg.rd_port == -1)
+ log_debug(" - write port %d: port group %s\n", i, cfg.def->port_groups[pcfg.port_group].names[0].c_str());
+ else
+ log_debug(" - write port %d: port group %s (shared with read port %d)\n", i, cfg.def->port_groups[pcfg.port_group].names[0].c_str(), pcfg.rd_port);
+
+ for (auto &it: pcfg.def->options)
+ log_debug(" - option %s %s\n", it.first.c_str(), log_const(it.second));
+ if (cfg.def->width_mode == WidthMode::PerPort) {
+ std::stringstream os;
+ for (int i = pcfg.def->min_wr_wide_log2; i <= pcfg.def->max_wr_wide_log2; i++)
+ os << " " << cfg.def->dbits[i];
+ std::string widths_s = os.str();
+ const char *note = "";
+ if (pcfg.rd_port != -1)
+ note = pcfg.def->width_tied ? " (tied)" : " (independent)";
+ log_debug(" - widths%s%s\n", widths_s.c_str(), note);
+ }
+ for (auto i: pcfg.emu_prio)
+ log_debug(" - emulate priority over write port %d\n", i);
+ }
+ for (int i = 0; i < GetSize(mem.rd_ports); i++) {
+ auto &pcfg = cfg.rd_ports[i];
+ if (pcfg.wr_port == -1)
+ log_debug(" - read port %d: port group %s\n", i, cfg.def->port_groups[pcfg.port_group].names[0].c_str());
+ else
+ log_debug(" - read port %d: port group %s (shared with write port %d)\n", i, cfg.def->port_groups[pcfg.port_group].names[0].c_str(), pcfg.wr_port);
+ for (auto &it: pcfg.def->options)
+ log_debug(" - option %s %s\n", it.first.c_str(), log_const(it.second));
+ if (cfg.def->width_mode == WidthMode::PerPort) {
+ std::stringstream os;
+ for (int i = pcfg.def->min_rd_wide_log2; i <= pcfg.def->max_rd_wide_log2; i++)
+ os << " " << cfg.def->dbits[i];
+ std::string widths_s = os.str();
+ const char *note = "";
+ if (pcfg.wr_port != -1)
+ note = pcfg.def->width_tied ? " (tied)" : " (independent)";
+ log_debug(" - widths%s%s\n", widths_s.c_str(), note);
+ }
+ if (pcfg.emu_sync)
+ log_debug(" - emulate data register\n");
+ if (pcfg.emu_en)
+ log_debug(" - emulate clock enable\n");
+ if (pcfg.emu_arst)
+ log_debug(" - emulate async reset\n");
+ if (pcfg.emu_srst)
+ log_debug(" - emulate sync reset\n");
+ if (pcfg.emu_init)
+ log_debug(" - emulate init value\n");
+ if (pcfg.emu_srst_en_prio)
+ log_debug(" - emulate sync reset / enable priority\n");
+ for (auto i: pcfg.emu_trans)
+ log_debug(" - emulate transparency with write port %d\n", i);
+ }
+}
+
+// Go through memory attributes to determine user-requested mapping style.
+void MemMapping::determine_style() {
+ kind = RamKind::Auto;
+ style = "";
+ if (mem.get_bool_attribute(ID::lram)) {
+ kind = RamKind::Huge;
+ return;
+ }
+ for (auto attr: {ID::ram_block, ID::rom_block, ID::ram_style, ID::rom_style, ID::ramstyle, ID::romstyle, ID::syn_ramstyle, ID::syn_romstyle}) {
+ if (mem.has_attribute(attr)) {
+ Const val = mem.attributes.at(attr);
+ if (val == 1) {
+ kind = RamKind::NotLogic;
+ return;
+ }
+ std::string val_s = val.decode_string();
+ for (auto &c: val_s)
+ c = std::tolower(c);
+ if (val_s == "auto") {
+ // Nothing.
+ } else if (val_s == "logic" || val_s == "registers") {
+ kind = RamKind::Logic;
+ } else if (val_s == "distributed") {
+ kind = RamKind::Distributed;
+ } else if (val_s == "block" || val_s == "block_ram" || val_s == "ebr") {
+ kind = RamKind::Block;
+ } else if (val_s == "huge" || val_s == "ultra") {
+ kind = RamKind::Huge;
+ } else {
+ kind = RamKind::NotLogic;
+ style = val_s;
+ }
+ return;
+ }
+ }
+ if (mem.get_bool_attribute(ID::logic_block))
+ kind = RamKind::Logic;
+}
+
+// Determine whether the memory can be mapped entirely to soft logic.
+bool MemMapping::determine_logic_ok() {
+ if (kind != RamKind::Auto && kind != RamKind::Logic)
+ return false;
+ // Memory is mappable entirely to soft logic iff all its write ports are in the same clock domain.
+ if (mem.wr_ports.empty())
+ return true;
+ for (auto &port: mem.wr_ports) {
+ if (!port.clk_enable)
+ return false;
+ if (port.clk != mem.wr_ports[0].clk)
+ return false;
+ if (port.clk_polarity != mem.wr_ports[0].clk_polarity)
+ return false;
+ }
+ return true;
+}
+
+// Apply RAM kind restrictions (logic/distributed/block/huge), if any.
+bool MemMapping::check_ram_kind(const Ram &ram) {
+ if (style != "")
+ return true;
+ if (ram.kind == kind)
+ return true;
+ if (kind == RamKind::Auto || kind == RamKind::NotLogic) {
+ if (ram.kind == RamKind::Distributed && opts.no_auto_distributed)
+ return false;
+ if (ram.kind == RamKind::Block && opts.no_auto_block)
+ return false;
+ if (ram.kind == RamKind::Huge && opts.no_auto_huge)
+ return false;
+ return true;
+ }
+ return false;
+}
+
+// Apply specific RAM style restrictions, if any.
+bool MemMapping::check_ram_style(const Ram &ram) {
+ if (style == "")
+ return true;
+ for (auto &s: ram.style)
+ if (s == style)
+ return true;
+ return false;
+}
+
+// Handle memory initializer restrictions, if any.
+bool MemMapping::check_init(const Ram &ram) {
+ bool has_nonx = false;
+ bool has_one = false;
+
+ for (auto &init: mem.inits) {
+ if (init.data.is_fully_undef())
+ continue;
+ has_nonx = true;
+ for (auto bit: init.data)
+ if (bit == State::S1)
+ has_one = true;
+ }
+
+ switch (ram.init) {
+ case MemoryInitKind::None:
+ return !has_nonx;
+ case MemoryInitKind::Zero:
+ return !has_one;
+ default:
+ return true;
+ }
+}
+
+bool apply_clock(MemConfig &cfg, const PortVariant &def, SigBit clk, bool clk_polarity) {
+ if (def.clk_shared == -1)
+ return true;
+ auto &cdef = cfg.def->shared_clocks[def.clk_shared];
+ auto &ccfg = cfg.shared_clocks[def.clk_shared];
+ if (cdef.anyedge) {
+ if (!ccfg.used) {
+ ccfg.used = true;
+ ccfg.clk = clk;
+ ccfg.polarity = clk_polarity;
+ return true;
+ } else {
+ return ccfg.clk == clk && ccfg.polarity == clk_polarity;
+ }
+ } else {
+ bool invert = clk_polarity ^ (def.clk_pol == ClkPolKind::Posedge);
+ if (!ccfg.used) {
+ ccfg.used = true;
+ ccfg.clk = clk;
+ ccfg.invert = invert;
+ return true;
+ } else {
+ return ccfg.clk == clk && ccfg.invert == invert;
+ }
+ }
+}
+
+// Perform write port assignment, validating clock options as we go.
+void MemMapping::assign_wr_ports() {
+ for (auto &port: mem.wr_ports) {
+ if (!port.clk_enable) {
+ // Async write ports not supported.
+ cfgs.clear();
+ return;
+ }
+ MemConfigs new_cfgs;
+ for (auto &cfg: cfgs) {
+ for (int pgi = 0; pgi < GetSize(cfg.def->port_groups); pgi++) {
+ auto &pg = cfg.def->port_groups[pgi];
+ // Make sure the target port group still has a free port.
+ int used = 0;
+ for (auto &oport: cfg.wr_ports)
+ if (oport.port_group == pgi)
+ used++;
+ if (used >= GetSize(pg.names))
+ continue;
+ for (int pvi = 0; pvi < GetSize(pg.variants); pvi++) {
+ auto &def = pg.variants[pvi];
+ // Make sure the target is a write port.
+ if (def.kind == PortKind::Ar || def.kind == PortKind::Sr)
+ continue;
+ MemConfig new_cfg = cfg;
+ WrPortConfig pcfg;
+ pcfg.rd_port = -1;
+ pcfg.port_group = pgi;
+ pcfg.port_variant = pvi;
+ pcfg.def = &def;
+ if (!apply_clock(new_cfg, def, port.clk, port.clk_polarity))
+ continue;
+ new_cfg.wr_ports.push_back(pcfg);
+ new_cfgs.push_back(new_cfg);
+ }
+ }
+ }
+ cfgs = new_cfgs;
+ }
+}
+
+// Perform read port assignment, validating clock and rden options as we go.
+void MemMapping::assign_rd_ports() {
+ for (int pidx = 0; pidx < GetSize(mem.rd_ports); pidx++) {
+ auto &port = mem.rd_ports[pidx];
+ MemConfigs new_cfgs;
+ for (auto &cfg: cfgs) {
+ // First pass: read port not shared with a write port.
+ for (int pgi = 0; pgi < GetSize(cfg.def->port_groups); pgi++) {
+ auto &pg = cfg.def->port_groups[pgi];
+ // Make sure the target port group has a port not used up by write ports.
+ // Overuse by other read ports is not a problem — this will just result
+ // in memory duplication.
+ int used = 0;
+ for (auto &oport: cfg.wr_ports)
+ if (oport.port_group == pgi)
+ used++;
+ if (used >= GetSize(pg.names))
+ continue;
+ for (int pvi = 0; pvi < GetSize(pg.variants); pvi++) {
+ auto &def = pg.variants[pvi];
+ // Make sure the target is a read port.
+ if (def.kind == PortKind::Sw)
+ continue;
+ // If mapping an async port, accept only async defs.
+ if (!port.clk_enable) {
+ if (def.kind == PortKind::Sr || def.kind == PortKind::Srsw)
+ continue;
+ }
+ MemConfig new_cfg = cfg;
+ RdPortConfig pcfg;
+ pcfg.wr_port = -1;
+ pcfg.port_group = pgi;
+ pcfg.port_variant = pvi;
+ pcfg.def = &def;
+ if (def.kind == PortKind::Sr || def.kind == PortKind::Srsw) {
+ pcfg.emu_sync = false;
+ if (!apply_clock(new_cfg, def, port.clk, port.clk_polarity))
+ continue;
+ // Decide if rden is usable.
+ if (port.en != State::S1) {
+ if (def.clk_en) {
+ pcfg.rd_en_to_clk_en = true;
+ } else {
+ pcfg.emu_en = !def.rd_en;
+ }
+ }
+ } else {
+ pcfg.emu_sync = port.clk_enable;
+ }
+ new_cfg.rd_ports.push_back(pcfg);
+ new_cfgs.push_back(new_cfg);
+ }
+ }
+ // Second pass: read port shared with a write port.
+ for (int wpidx = 0; wpidx < GetSize(mem.wr_ports); wpidx++) {
+ auto &wport = mem.wr_ports[wpidx];
+ auto &wpcfg = cfg.wr_ports[wpidx];
+ auto &def = *wpcfg.def;
+ // Make sure the write port is not yet shared.
+ if (wpcfg.rd_port != -1)
+ continue;
+ // Make sure the target is a read port.
+ if (def.kind == PortKind::Sw)
+ continue;
+ // Validate address compatibility.
+ if (!addr_compatible(wpidx, pidx))
+ continue;
+ // Validate clock compatibility, if needed.
+ if (def.kind == PortKind::Srsw) {
+ if (!port.clk_enable)
+ continue;
+ if (port.clk != wport.clk)
+ continue;
+ if (port.clk_polarity != wport.clk_polarity)
+ continue;
+ }
+ // Okay, let's fill it in.
+ MemConfig new_cfg = cfg;
+ new_cfg.wr_ports[wpidx].rd_port = pidx;
+ RdPortConfig pcfg;
+ pcfg.wr_port = wpidx;
+ pcfg.port_group = wpcfg.port_group;
+ pcfg.port_variant = wpcfg.port_variant;
+ pcfg.def = wpcfg.def;
+ pcfg.emu_sync = port.clk_enable && def.kind == PortKind::Arsw;
+ // For srsw, check rden capability.
+ if (def.kind == PortKind::Srsw) {
+ bool trans = port.transparency_mask[wpidx];
+ bool col_x = port.collision_x_mask[wpidx];
+ if (def.rdwr == RdWrKind::NoChange) {
+ if (!get_wr_excludes_rd(wpidx, pidx)) {
+ if (!trans && !col_x)
+ continue;
+ if (trans)
+ pcfg.emu_trans.push_back(wpidx);
+ new_cfg.wr_ports[wpidx].force_uniform = true;
+ }
+ if (port.en != State::S1) {
+ if (def.clk_en) {
+ pcfg.rd_en_to_clk_en = true;
+ } else {
+ pcfg.emu_en = !def.rd_en;
+ }
+ }
+ } else {
+ if (!col_x && !trans && def.rdwr != RdWrKind::Old)
+ continue;
+ if (trans) {
+ if (def.rdwr != RdWrKind::New && def.rdwr != RdWrKind::NewOnly)
+ pcfg.emu_trans.push_back(wpidx);
+ }
+ if (def.rdwr == RdWrKind::NewOnly) {
+ if (!get_wr_excludes_rd(wpidx, pidx))
+ new_cfg.wr_ports[wpidx].force_uniform = true;
+ }
+ if (port.en != State::S1) {
+ if (def.clk_en) {
+ if (get_wr_implies_rd(wpidx, pidx)) {
+ pcfg.rd_en_to_clk_en = true;
+ } else {
+ pcfg.emu_en = !def.rd_en;
+ }
+ } else {
+ pcfg.emu_en = !def.rd_en;
+ }
+ }
+ }
+ }
+ new_cfg.rd_ports.push_back(pcfg);
+ new_cfgs.push_back(new_cfg);
+ }
+ }
+ cfgs = new_cfgs;
+ }
+}
+
+// Validate transparency restrictions, determine where to add soft transparency logic.
+void MemMapping::handle_trans() {
+ if (mem.emulate_read_first_ok()) {
+ MemConfigs new_cfgs;
+ for (auto &cfg: cfgs) {
+ new_cfgs.push_back(cfg);
+ bool ok = true;
+ // Using this trick will break read-write port sharing.
+ for (auto &pcfg: cfg.rd_ports)
+ if (pcfg.wr_port != -1)
+ ok = false;
+ if (ok) {
+ cfg.emu_read_first = true;
+ new_cfgs.push_back(cfg);
+ }
+ }
+ cfgs = new_cfgs;
+ }
+ for (int rpidx = 0; rpidx < GetSize(mem.rd_ports); rpidx++) {
+ auto &rport = mem.rd_ports[rpidx];
+ if (!rport.clk_enable)
+ continue;
+ for (int wpidx = 0; wpidx < GetSize(mem.wr_ports); wpidx++) {
+ auto &wport = mem.wr_ports[wpidx];
+ if (!wport.clk_enable)
+ continue;
+ if (rport.clk != wport.clk)
+ continue;
+ if (rport.clk_polarity != wport.clk_polarity)
+ continue;
+ // If we got this far, we have a transparency restriction
+ // to uphold.
+ MemConfigs new_cfgs;
+ for (auto &cfg: cfgs) {
+ auto &rpcfg = cfg.rd_ports[rpidx];
+ auto &wpcfg = cfg.wr_ports[wpidx];
+ // The transparency relation for shared ports already handled while assigning them.
+ if (rpcfg.wr_port == wpidx) {
+ new_cfgs.push_back(cfg);
+ continue;
+ }
+ if (rport.collision_x_mask[wpidx] && !cfg.emu_read_first) {
+ new_cfgs.push_back(cfg);
+ continue;
+ }
+ bool transparent = rport.transparency_mask[wpidx] || cfg.emu_read_first;
+ if (rpcfg.emu_sync) {
+ // For async read port, just add the transparency logic
+ // if necessary.
+ if (transparent)
+ rpcfg.emu_trans.push_back(wpidx);
+ new_cfgs.push_back(cfg);
+ } else {
+ // Otherwise, split through the relevant wrtrans caps.
+ // For non-transparent ports, the cap needs to be present.
+ // For transparent ports, we can emulate transparency
+ // even without a direct cap.
+ bool found = false;
+ for (auto &tdef: wpcfg.def->wrtrans) {
+ // Check if the target matches.
+ if (tdef.target_kind == WrTransTargetKind::Group && rpcfg.port_group != tdef.target_group)
+ continue;
+ // Check if the transparency kind is acceptable.
+ if (transparent) {
+ if (tdef.kind == WrTransKind::Old)
+ continue;
+ } else {
+ if (tdef.kind != WrTransKind::Old)
+ continue;
+ }
+ // Okay, we can use this cap.
+ new_cfgs.push_back(cfg);
+ found = true;
+ break;
+ }
+ if (!found && transparent) {
+ // If the port pair is transparent, but no cap was
+ // found, use emulation.
+ rpcfg.emu_trans.push_back(wpidx);
+ new_cfgs.push_back(cfg);
+ }
+ }
+ }
+ cfgs = new_cfgs;
+ }
+ }
+}
+
+// Determine where to add soft priority logic.
+void MemMapping::handle_priority() {
+ for (int p1idx = 0; p1idx < GetSize(mem.wr_ports); p1idx++) {
+ for (int p2idx = 0; p2idx < GetSize(mem.wr_ports); p2idx++) {
+ auto &port2 = mem.wr_ports[p2idx];
+ if (!port2.priority_mask[p1idx])
+ continue;
+ for (auto &cfg: cfgs) {
+ auto &p1cfg = cfg.rd_ports[p1idx];
+ auto &p2cfg = cfg.wr_ports[p2idx];
+ bool found = false;
+ for (auto &pgi: p2cfg.def->wrprio) {
+ if (pgi == p1cfg.port_group) {
+ found = true;
+ break;
+ }
+ }
+ // If no cap was found, emulate.
+ if (!found)
+ p2cfg.emu_prio.push_back(p1idx);
+ }
+ }
+ }
+}
+
+bool is_all_zero(const Const &val) {
+ for (auto bit: val.bits)
+ if (bit == State::S1)
+ return false;
+ return true;
+}
+
+// Determine where to add soft init value / reset logic.
+void MemMapping::handle_rd_rst() {
+ for (auto &cfg: cfgs) {
+ for (int pidx = 0; pidx < GetSize(mem.rd_ports); pidx++) {
+ auto &port = mem.rd_ports[pidx];
+ auto &pcfg = cfg.rd_ports[pidx];
+ // Only sync ports are relevant.
+ // If emulated by async port or we already emulate CE, init will be
+ // included for free.
+ if (!port.clk_enable || pcfg.emu_sync || pcfg.emu_en)
+ continue;
+ switch (pcfg.def->rdinitval) {
+ case ResetValKind::None:
+ pcfg.emu_init = !port.init_value.is_fully_undef();
+ break;
+ case ResetValKind::Zero:
+ pcfg.emu_init = !is_all_zero(port.init_value);
+ break;
+ default:
+ break;
+ }
+ Const init_val = port.init_value;
+ if (port.arst != State::S0) {
+ switch (pcfg.def->rdarstval) {
+ case ResetValKind::None:
+ pcfg.emu_arst = true;
+ break;
+ case ResetValKind::Zero:
+ pcfg.emu_arst = !is_all_zero(port.arst_value);
+ break;
+ case ResetValKind::Init:
+ if (init_val.is_fully_undef())
+ init_val = port.arst_value;
+ pcfg.emu_arst = init_val != port.arst_value;
+ break;
+ default:
+ break;
+ }
+ }
+ if (port.srst != State::S0) {
+ switch (pcfg.def->rdsrstval) {
+ case ResetValKind::None:
+ pcfg.emu_srst = true;
+ break;
+ case ResetValKind::Zero:
+ pcfg.emu_srst = !is_all_zero(port.srst_value);
+ break;
+ case ResetValKind::Init:
+ if (init_val.is_fully_undef())
+ init_val = port.srst_value;
+ pcfg.emu_srst = init_val != port.srst_value;
+ break;
+ default:
+ break;
+ }
+ if (!pcfg.emu_srst && pcfg.def->rdsrst_block_wr && pcfg.wr_port != -1) {
+ if (!get_wr_excludes_srst(pcfg.wr_port, pidx))
+ pcfg.emu_srst = true;
+ }
+ if (!pcfg.emu_srst && port.en != State::S1) {
+ if (port.ce_over_srst) {
+ switch (pcfg.def->rdsrstmode) {
+ case SrstKind::Ungated:
+ pcfg.emu_srst_en_prio = true;
+ break;
+ case SrstKind::GatedClkEn:
+ pcfg.emu_srst_en_prio = !pcfg.rd_en_to_clk_en;
+ break;
+ case SrstKind::GatedRdEn:
+ break;
+ default:
+ log_assert(0);
+ }
+ } else {
+ switch (pcfg.def->rdsrstmode) {
+ case SrstKind::Ungated:
+ break;
+ case SrstKind::GatedClkEn:
+ if (pcfg.rd_en_to_clk_en) {
+ if (pcfg.def->rd_en) {
+ pcfg.rd_en_to_clk_en = false;
+ } else {
+ pcfg.emu_srst_en_prio = true;
+ }
+ }
+ break;
+ case SrstKind::GatedRdEn:
+ pcfg.emu_srst_en_prio = true;
+ break;
+ default:
+ log_assert(0);
+ }
+ }
+ }
+ } else {
+ if (pcfg.def->rd_en && pcfg.def->rdwr == RdWrKind::NoChange && pcfg.wr_port != -1) {
+ pcfg.rd_en_to_clk_en = false;
+ }
+ }
+ }
+ }
+}
+
+void MemMapping::score_emu_ports() {
+ for (auto &cfg: cfgs) {
+ std::vector<int> port_usage_wr(cfg.def->port_groups.size());
+ std::vector<int> port_usage_rd(cfg.def->port_groups.size());
+ int score = 0;
+ // 3 points for every write port if we need to do read-first emulation.
+ if (cfg.emu_read_first)
+ score += 3 * GetSize(cfg.wr_ports);
+ for (auto &pcfg: cfg.wr_ports) {
+ // 1 point for every priority relation we need to fix up.
+ // This is just a gate for every distinct wren pair.
+ score += GetSize(pcfg.emu_prio);
+ port_usage_wr[pcfg.port_group]++;
+ }
+ for (auto &pcfg: cfg.rd_ports) {
+ // 3 points for every soft transparency logic instance. This involves
+ // registers and other major mess.
+ score += 3 * GetSize(pcfg.emu_trans);
+ // 3 points for CE soft logic. Likewise involves registers.
+ // If we already do this, subsumes any init/srst/arst emulation.
+ if (pcfg.emu_en)
+ score += 3;
+ // 2 points for soft init value / reset logic: involves single bit
+ // register and some muxes.
+ if (pcfg.emu_init)
+ score += 2;
+ if (pcfg.emu_arst)
+ score += 2;
+ if (pcfg.emu_srst)
+ score += 2;
+ // 1 point for wrong srst/en priority (fixed with a single gate).
+ if (pcfg.emu_srst_en_prio)
+ score++;
+ // 1 point for every non-shared read port used, as a tiebreaker
+ // to prefer single-port configs.
+ if (pcfg.wr_port == -1) {
+ score++;
+ port_usage_rd[pcfg.port_group]++;
+ }
+ }
+ cfg.score_emu = score;
+ int repl_port = 1;
+ for (int i = 0; i < GetSize(cfg.def->port_groups); i++) {
+ int space = GetSize(cfg.def->port_groups[i].names) - port_usage_wr[i];
+ log_assert(space >= 0);
+ if (port_usage_rd[i] > 0) {
+ log_assert(space > 0);
+ int usage = port_usage_rd[i];
+ int cur = (usage + space - 1) / space;
+ if (cur > repl_port)
+ repl_port = cur;
+ }
+ }
+ cfg.repl_port = repl_port;
+ }
+}
+
+void MemMapping::handle_geom() {
+ std::vector<int> wren_size;
+ for (auto &port: mem.wr_ports) {
+ SigSpec en = port.en;
+ en.sort_and_unify();
+ wren_size.push_back(GetSize(en));
+ }
+ for (auto &cfg: cfgs) {
+ // First, create a set of "byte boundaries": the bit positions in source memory word
+ // that have write enable different from the previous bit in any write port.
+ // Bit 0 is considered to be a byte boundary as well.
+ // Likewise, create a set of "word boundaries" that are like above, but only for write ports
+ // with the "force uniform" flag set.
+ std::vector<bool> byte_boundary(mem.width, false);
+ std::vector<bool> word_boundary(mem.width, false);
+ byte_boundary[0] = true;
+ for (int pidx = 0; pidx < GetSize(mem.wr_ports); pidx++) {
+ auto &port = mem.wr_ports[pidx];
+ auto &pcfg = cfg.wr_ports[pidx];
+ if (pcfg.force_uniform)
+ word_boundary[0] = true;
+ for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
+ for (int i = 1; i < mem.width; i++) {
+ int pos = sub * mem.width + i;
+ if (port.en[pos] != port.en[pos-1]) {
+ byte_boundary[i] = true;
+ if (pcfg.force_uniform)
+ word_boundary[i] = true;
+ }
+ }
+ }
+ }
+ bool got_config = false;
+ int best_cost = 0;
+ int byte_width_log2 = 0;
+ for (int i = 0; i < GetSize(cfg.def->dbits); i++)
+ if (cfg.def->byte >= cfg.def->dbits[i])
+ byte_width_log2 = i;
+ if (cfg.def->byte == 0)
+ byte_width_log2 = GetSize(cfg.def->dbits) - 1;
+ pool<int> no_wide_bits;
+ // Determine which of the source address bits involved in wide ports
+ // are "uniform". Bits are considered uniform if, when a port is widened through
+ // them, the write enables are the same for both values of the bit.
+ int max_wr_wide_log2 = 0;
+ for (auto &port: mem.wr_ports)
+ if (port.wide_log2 > max_wr_wide_log2)
+ max_wr_wide_log2 = port.wide_log2;
+ int max_wide_log2 = max_wr_wide_log2;
+ for (auto &port: mem.rd_ports)
+ if (port.wide_log2 > max_wide_log2)
+ max_wide_log2 = port.wide_log2;
+ int wide_nu_start = max_wide_log2;
+ int wide_nu_end = max_wr_wide_log2;
+ for (int i = 0; i < GetSize(mem.wr_ports); i++) {
+ auto &port = mem.wr_ports[i];
+ auto &pcfg = cfg.wr_ports[i];
+ for (int j = 0; j < port.wide_log2; j++) {
+ bool uniform = true;
+ // If write enables don't match, mark bit as non-uniform.
+ for (int k = 0; k < (1 << port.wide_log2); k += 2 << j)
+ if (port.en.extract(k * mem.width, mem.width << j) != port.en.extract((k + (1 << j)) * mem.width, mem.width << j))
+ uniform = false;
+ if (!uniform) {
+ if (pcfg.force_uniform) {
+ for (int k = j; k < port.wide_log2; k++)
+ no_wide_bits.insert(k);
+ }
+ if (j < wide_nu_start)
+ wide_nu_start = j;
+ break;
+ }
+ }
+ if (pcfg.def->width_tied && pcfg.rd_port != -1) {
+ // If:
+ //
+ // - the write port is merged with a read port
+ // - the read port is wider than the write port
+ // - read and write widths are tied
+ //
+ // then we will have to artificially widen the write
+ // port to the width of the read port, and emulate
+ // a narrower write path by use of write enables,
+ // which will definitely be non-uniform over the added
+ // bits.
+ auto &rport = mem.rd_ports[pcfg.rd_port];
+ if (rport.wide_log2 > port.wide_log2) {
+ if (port.wide_log2 < wide_nu_start)
+ wide_nu_start = port.wide_log2;
+ if (rport.wide_log2 > wide_nu_end)
+ wide_nu_end = rport.wide_log2;
+ if (pcfg.force_uniform) {
+ for (int k = port.wide_log2; k < rport.wide_log2; k++)
+ no_wide_bits.insert(k);
+ }
+ }
+ }
+ }
+ // Iterate over base widths.
+ for (int base_width_log2 = 0; base_width_log2 < GetSize(cfg.def->dbits); base_width_log2++) {
+ // Now, see how many data bits we actually have available.
+ // This is usually dbits[base_width_log2], but could be smaller if we
+ // ran afoul of a max width limitation. Configurations where this
+ // happens are not useful, unless we need it to satisfy a *minimum*
+ // width limitation.
+ int unit_width_log2 = base_width_log2;
+ for (auto &pcfg: cfg.wr_ports)
+ if (unit_width_log2 > pcfg.def->max_wr_wide_log2)
+ unit_width_log2 = pcfg.def->max_wr_wide_log2;
+ for (auto &pcfg: cfg.rd_ports)
+ if (unit_width_log2 > pcfg.def->max_rd_wide_log2)
+ unit_width_log2 = pcfg.def->max_rd_wide_log2;
+ if (unit_width_log2 != base_width_log2 && got_config)
+ break;
+ int unit_width = cfg.def->dbits[unit_width_log2];
+ // Also determine effective byte width (the granularity of write enables).
+ int effective_byte = cfg.def->byte;
+ if (effective_byte == 0 || effective_byte > unit_width)
+ effective_byte = unit_width;
+ if (mem.wr_ports.empty())
+ effective_byte = 1;
+ log_assert(unit_width % effective_byte == 0);
+ // Create the swizzle pattern.
+ std::vector<int> swizzle;
+ for (int i = 0; i < mem.width; i++) {
+ if (word_boundary[i])
+ while (GetSize(swizzle) % unit_width)
+ swizzle.push_back(-1);
+ else if (byte_boundary[i])
+ while (GetSize(swizzle) % effective_byte)
+ swizzle.push_back(-1);
+ swizzle.push_back(i);
+ }
+ if (word_boundary[0])
+ while (GetSize(swizzle) % unit_width)
+ swizzle.push_back(-1);
+ else
+ while (GetSize(swizzle) % effective_byte)
+ swizzle.push_back(-1);
+ // Now evaluate the configuration, then keep adding more hard wide bits
+ // and evaluating.
+ int hard_wide_mask = 0;
+ int hard_wide_num = 0;
+ bool byte_failed = false;
+ while (1) {
+ // Check if all min width constraints are satisfied.
+ // Only check these constraints for write ports with width below
+ // byte width — for other ports, we can emulate narrow width with
+ // a larger one.
+ bool min_width_ok = true;
+ int min_width_bit = wide_nu_start;
+ for (int pidx = 0; pidx < GetSize(mem.wr_ports); pidx++) {
+ auto &port = mem.wr_ports[pidx];
+ int w = base_width_log2;
+ for (int i = 0; i < port.wide_log2; i++)
+ if (hard_wide_mask & 1 << i)
+ w++;
+ if (w < cfg.wr_ports[pidx].def->min_wr_wide_log2 && w < byte_width_log2) {
+ min_width_ok = false;
+ if (min_width_bit > port.wide_log2)
+ min_width_bit = port.wide_log2;
+ }
+ }
+ if (min_width_ok) {
+ int emu_wide_bits = max_wide_log2 - hard_wide_num;
+ int mult_wide = 1 << emu_wide_bits;
+ int addrs = 1 << (cfg.def->abits - base_width_log2 + emu_wide_bits);
+ int min_addr = mem.start_offset / addrs;
+ int max_addr = (mem.start_offset + mem.size - 1) / addrs;
+ int mult_a = max_addr - min_addr + 1;
+ int bits = mult_a * mult_wide * GetSize(swizzle);
+ int repl = (bits + unit_width - 1) / unit_width;
+ int score_demux = 0;
+ for (int i = 0; i < GetSize(mem.wr_ports); i++) {
+ auto &port = mem.wr_ports[i];
+ int w = emu_wide_bits;
+ for (int i = 0; i < port.wide_log2; i++)
+ if (!(hard_wide_mask & 1 << i))
+ w--;
+ if (w || mult_a != 1)
+ score_demux += (mult_a << w) * wren_size[i];
+ }
+ int score_mux = 0;
+ for (auto &port: mem.rd_ports) {
+ int w = emu_wide_bits;
+ for (int i = 0; i < port.wide_log2; i++)
+ if (!(hard_wide_mask & 1 << i))
+ w--;
+ score_mux += ((mult_a << w) - 1) * GetSize(port.data);
+ }
+ double cost = (cfg.def->cost - cfg.def->widthscale) * repl * cfg.repl_port;
+ cost += cfg.def->widthscale * mult_a * mult_wide * mem.width / unit_width * cfg.repl_port;
+ cost += score_mux * FACTOR_MUX;
+ cost += score_demux * FACTOR_DEMUX;
+ cost += cfg.score_emu * FACTOR_EMU;
+ if (!got_config || cost < best_cost) {
+ cfg.base_width_log2 = base_width_log2;
+ cfg.unit_width_log2 = unit_width_log2;
+ cfg.swizzle = swizzle;
+ cfg.hard_wide_mask = hard_wide_mask;
+ cfg.emu_wide_mask = ((1 << max_wide_log2) - 1) & ~hard_wide_mask;
+ cfg.repl_d = repl;
+ cfg.score_demux = score_demux;
+ cfg.score_mux = score_mux;
+ cfg.cost = cost;
+ best_cost = cost;
+ got_config = true;
+ }
+ }
+ if (cfg.def->width_mode != WidthMode::PerPort)
+ break;
+ // Now, pick the next bit to add to the hard wide mask.
+next_hw:
+ int scan_from;
+ int scan_to;
+ bool retry = false;
+ if (!min_width_ok) {
+ // If we still haven't met the minimum width limits,
+ // add the highest one that will be useful for working
+ // towards all unmet limits.
+ scan_from = min_width_bit;
+ scan_to = 0;
+ // If the relevant write port is not wide, it's impossible.
+ } else if (byte_failed) {
+ // If we already failed with uniformly-written bits only,
+ // go with uniform bits that are only involved in reads.
+ scan_from = max_wide_log2;
+ scan_to = wide_nu_end;
+ } else if (base_width_log2 + hard_wide_num < byte_width_log2) {
+ // If we still need uniform bits, prefer the low ones.
+ scan_from = wide_nu_start;
+ scan_to = 0;
+ retry = true;
+ } else {
+ scan_from = max_wide_log2;
+ scan_to = 0;
+ }
+ int bit = scan_from - 1;
+ while (1) {
+ if (bit < scan_to) {
+hw_bit_failed:
+ if (retry) {
+ byte_failed = true;
+ goto next_hw;
+ } else {
+ goto bw_done;
+ }
+ }
+ if (!(hard_wide_mask & 1 << bit) && !no_wide_bits.count(bit))
+ break;
+ bit--;
+ }
+ int new_hw_mask = hard_wide_mask | 1 << bit;
+ // Check if all max width constraints are satisfied.
+ for (int pidx = 0; pidx < GetSize(mem.wr_ports); pidx++) {
+ auto &port = mem.wr_ports[pidx];
+ int w = base_width_log2;
+ for (int i = 0; i < port.wide_log2; i++)
+ if (new_hw_mask & 1 << i)
+ w++;
+ if (w > cfg.wr_ports[pidx].def->max_wr_wide_log2) {
+ goto hw_bit_failed;
+ }
+ }
+ for (int pidx = 0; pidx < GetSize(mem.rd_ports); pidx++) {
+ auto &port = mem.rd_ports[pidx];
+ int w = base_width_log2;
+ for (int i = 0; i < port.wide_log2; i++)
+ if (new_hw_mask & 1 << i)
+ w++;
+ if (w > cfg.rd_ports[pidx].def->max_rd_wide_log2) {
+ goto hw_bit_failed;
+ }
+ }
+ // Bit ok, commit.
+ hard_wide_mask = new_hw_mask;
+ hard_wide_num++;
+ }
+bw_done:;
+ }
+ log_assert(got_config);
+ }
+}
+
+void MemMapping::prune_post_geom() {
+ std::vector<bool> keep;
+ dict<std::string, int> rsrc;
+ for (int i = 0; i < GetSize(cfgs); i++) {
+ auto &cfg = cfgs[i];
+ std::string key = cfg.def->resource_name;
+ if (key.empty()) {
+ switch (cfg.def->kind) {
+ case RamKind::Distributed:
+ key = "[distributed]";
+ break;
+ case RamKind::Block:
+ key = "[block]";
+ break;
+ case RamKind::Huge:
+ key = "[huge]";
+ break;
+ default:
+ break;
+ }
+ }
+ auto it = rsrc.find(key);
+ if (it == rsrc.end()) {
+ rsrc[key] = i;
+ keep.push_back(true);
+ } else {
+ auto &ocfg = cfgs[it->second];
+ if (cfg.cost < ocfg.cost) {
+ keep[it->second] = false;
+ it->second = i;
+ keep.push_back(true);
+ } else {
+ keep.push_back(false);
+ }
+ }
+ }
+ MemConfigs new_cfgs;
+ for (int i = 0; i < GetSize(cfgs); i++)
+ if (keep[i])
+ new_cfgs.push_back(cfgs[i]);
+ cfgs = new_cfgs;
+}
+
+Swizzle gen_swizzle(const Mem &mem, const MemConfig &cfg, int sw_wide_log2, int hw_wide_log2) {
+ Swizzle res;
+
+ std::vector<int> emu_wide_bits;
+ std::vector<int> hard_wide_bits;
+ for (int i = 0; i < ceil_log2(mem.size); i++) {
+ if (cfg.emu_wide_mask & 1 << i)
+ emu_wide_bits.push_back(i);
+ else if (GetSize(hard_wide_bits) < hw_wide_log2 - cfg.base_width_log2)
+ hard_wide_bits.push_back(i);
+ }
+ for (int x : hard_wide_bits)
+ if (x >= sw_wide_log2)
+ res.addr_mux_bits.push_back(x);
+ for (int x : emu_wide_bits)
+ if (x >= sw_wide_log2)
+ res.addr_mux_bits.push_back(x);
+
+ res.addr_shift = cfg.def->abits - cfg.base_width_log2 + GetSize(emu_wide_bits);
+ res.addr_start = mem.start_offset & ~((1 << res.addr_shift) - 1);
+ res.addr_end = ((mem.start_offset + mem.size - 1) | ((1 << res.addr_shift) - 1)) + 1;
+ int hnum = (res.addr_end - res.addr_start) >> res.addr_shift;
+ int unit_width = cfg.def->dbits[cfg.unit_width_log2];
+
+ for (int rd = 0; rd < cfg.repl_d; rd++) {
+ std::vector<SwizzleBit> bits(cfg.def->dbits[hw_wide_log2]);
+ for (auto &bit: bits)
+ bit.valid = false;
+ res.bits.push_back(bits);
+ }
+
+ for (int hi = 0; hi < hnum; hi++) {
+ for (int ewi = 0; ewi < (1 << GetSize(emu_wide_bits)); ewi++) {
+ for (int hwi = 0; hwi < (1 << GetSize(hard_wide_bits)); hwi++) {
+ int mux_idx = 0;
+ int sub = 0;
+ int mib = 0;
+ int hbit_base = 0;
+ for (int i = 0; i < GetSize(hard_wide_bits); i++) {
+ if (hard_wide_bits[i] < sw_wide_log2) {
+ if (hwi & 1 << i)
+ sub |= 1 << hard_wide_bits[i];
+ } else {
+ if (hwi & 1 << i)
+ mux_idx |= 1 << mib;
+ mib++;
+ }
+ if (hwi & 1 << i)
+ hbit_base += cfg.def->dbits[i + cfg.base_width_log2];
+ }
+ for (int i = 0; i < GetSize(emu_wide_bits); i++) {
+ if (emu_wide_bits[i] < sw_wide_log2) {
+ if (ewi & 1 << i)
+ sub |= 1 << emu_wide_bits[i];
+ } else {
+ if (ewi & 1 << i)
+ mux_idx |= 1 << mib;
+ mib++;
+ }
+ }
+ mux_idx |= hi << mib;
+ int addr = res.addr_start + (hi << res.addr_shift);
+ for (int i = 0; i < GetSize(res.addr_mux_bits); i++)
+ if (mux_idx & 1 << i)
+ addr += 1 << res.addr_mux_bits[i];
+ for (int bit = 0; bit < GetSize(cfg.swizzle); bit++) {
+ if (cfg.swizzle[bit] == -1)
+ continue;
+ int rbit = bit + GetSize(cfg.swizzle) * (ewi + (hi << GetSize(emu_wide_bits)));
+ int rep = rbit / unit_width;
+ int hbit = hbit_base + rbit % unit_width;
+ auto &swz = res.bits[rep][hbit];
+ swz.valid = true;
+ swz.addr = addr;
+ swz.mux_idx = mux_idx;
+ swz.bit = cfg.swizzle[bit] + sub * mem.width;
+ }
+ }
+ }
+ }
+
+ return res;
+}
+
+void clean_undef(std::vector<State> &val) {
+ for (auto &bit : val)
+ if (bit != State::S1)
+ bit = State::S0;
+}
+
+std::vector<SigSpec> generate_demux(Mem &mem, int wpidx, const Swizzle &swz) {
+ auto &port = mem.wr_ports[wpidx];
+ std::vector<SigSpec> res;
+ int hi_bits = ceil_log2(swz.addr_end - swz.addr_start) - swz.addr_shift;
+ auto compressed = port.compress_en();
+ SigSpec sig_a = compressed.first;
+ SigSpec addr = port.addr;
+ if (GetSize(addr) > hi_bits + swz.addr_shift) {
+ int lo = mem.start_offset;
+ int hi = mem.start_offset + mem.size;
+ int bits = ceil_log2(hi);
+ for (int i = 0; i < bits; i++) {
+ int new_lo = lo;
+ if (lo & 1 << i)
+ new_lo -= 1 << i;
+ int new_hi = hi;
+ if (hi & 1 << i)
+ new_hi += 1 << i;
+ if (new_hi - new_lo > (1 << (hi_bits + swz.addr_shift)))
+ break;
+ lo = new_lo;
+ hi = new_hi;
+ }
+ SigSpec in_range = mem.module->And(NEW_ID, mem.module->Ge(NEW_ID, addr, lo), mem.module->Lt(NEW_ID, addr, hi));
+ sig_a = mem.module->Mux(NEW_ID, Const(State::S0, GetSize(sig_a)), sig_a, in_range);
+ }
+ addr.extend_u0(swz.addr_shift + hi_bits, false);
+ SigSpec sig_s;
+ for (int x : swz.addr_mux_bits)
+ sig_s.append(addr[x]);
+ for (int i = 0; i < hi_bits; i++)
+ sig_s.append(addr[swz.addr_shift + i]);
+ SigSpec sig_y;
+ if (GetSize(sig_s) == 0)
+ sig_y = sig_a;
+ else
+ sig_y = mem.module->Demux(NEW_ID, sig_a, sig_s);
+ for (int i = 0; i < ((swz.addr_end - swz.addr_start) >> swz.addr_shift); i++) {
+ for (int j = 0; j < (1 << GetSize(swz.addr_mux_bits)); j++) {
+ int hi = ((swz.addr_start >> swz.addr_shift) + i) & ((1 << hi_bits) - 1);
+ int pos = (hi << GetSize(swz.addr_mux_bits) | j) * GetSize(sig_a);
+ res.push_back(port.decompress_en(compressed.second, sig_y.extract(pos, GetSize(sig_a))));
+ }
+ }
+ return res;
+}
+
+std::vector<SigSpec> generate_mux(Mem &mem, int rpidx, const Swizzle &swz) {
+ auto &port = mem.rd_ports[rpidx];
+ std::vector<SigSpec> res;
+ int hi_bits = ceil_log2(swz.addr_end - swz.addr_start) - swz.addr_shift;
+ SigSpec sig_s;
+ SigSpec addr = port.addr;
+ addr.extend_u0(swz.addr_shift + hi_bits, false);
+ for (int x : swz.addr_mux_bits)
+ sig_s.append(addr[x]);
+ for (int i = 0; i < hi_bits; i++)
+ sig_s.append(addr[swz.addr_shift + i]);
+ if (GetSize(sig_s) == 0) {
+ return {port.data};
+ }
+ if (port.clk_enable) {
+ SigSpec new_sig_s = mem.module->addWire(NEW_ID, GetSize(sig_s));
+ mem.module->addDffe(NEW_ID, port.clk, port.en, sig_s, new_sig_s, port.clk_polarity);
+ sig_s = new_sig_s;
+ }
+ SigSpec sig_a = Const(State::Sx, GetSize(port.data) << hi_bits << GetSize(swz.addr_mux_bits));
+ for (int i = 0; i < ((swz.addr_end - swz.addr_start) >> swz.addr_shift); i++) {
+ for (int j = 0; j < (1 << GetSize(swz.addr_mux_bits)); j++) {
+ SigSpec sig = mem.module->addWire(NEW_ID, GetSize(port.data));
+ int hi = ((swz.addr_start >> swz.addr_shift) + i) & ((1 << hi_bits) - 1);
+ int pos = (hi << GetSize(swz.addr_mux_bits) | j) * GetSize(port.data);
+ for (int k = 0; k < GetSize(port.data); k++)
+ sig_a[pos + k] = sig[k];
+ res.push_back(sig);
+ }
+ }
+ mem.module->addBmux(NEW_ID, sig_a, sig_s, port.data);
+ return res;
+}
+
+void MemMapping::emit_port(const MemConfig &cfg, std::vector<Cell*> &cells, const PortVariant &pdef, const char *name, int wpidx, int rpidx, const std::vector<int> &hw_addr_swizzle) {
+ for (auto &it: pdef.options)
+ for (auto cell: cells)
+ cell->setParam(stringf("\\PORT_%s_OPTION_%s", name, it.first.c_str()), it.second);
+ SigSpec addr = Const(State::Sx, cfg.def->abits);
+ int wide_log2 = 0, wr_wide_log2 = 0, rd_wide_log2 = 0;
+ SigSpec clk = State::S0;
+ SigSpec clk_en = State::S0;
+ bool clk_pol = true;
+ if (wpidx != -1) {
+ auto &wport = mem.wr_ports[wpidx];
+ clk = wport.clk;
+ clk_pol = wport.clk_polarity;
+ addr = wport.addr;
+ wide_log2 = wr_wide_log2 = wport.wide_log2;
+ if (rpidx != -1) {
+ auto &rport = mem.rd_ports[rpidx];
+ auto &rpcfg = cfg.rd_ports[rpidx];
+ rd_wide_log2 = rport.wide_log2;
+ if (rd_wide_log2 > wr_wide_log2)
+ wide_log2 = rd_wide_log2;
+ else
+ addr = rport.addr;
+ if (pdef.clk_en) {
+ if (rpcfg.rd_en_to_clk_en) {
+ if (pdef.rdwr == RdWrKind::NoChange) {
+ clk_en = mem.module->Or(NEW_ID, rport.en, mem.module->ReduceOr(NEW_ID, wport.en));
+ } else {
+ clk_en = rport.en;
+ }
+ } else {
+ clk_en = State::S1;
+ }
+ }
+ } else {
+ if (pdef.clk_en)
+ clk_en = State::S1;
+ }
+ } else if (rpidx != -1) {
+ auto &rport = mem.rd_ports[rpidx];
+ auto &rpcfg = cfg.rd_ports[rpidx];
+ if (rport.clk_enable) {
+ clk = rport.clk;
+ clk_pol = rport.clk_polarity;
+ }
+ addr = rport.addr;
+ wide_log2 = rd_wide_log2 = rport.wide_log2;
+ if (pdef.clk_en) {
+ if (rpcfg.rd_en_to_clk_en)
+ clk_en = rport.en;
+ else
+ clk_en = State::S1;
+ }
+
+ }
+ addr = worker.sigmap_xmux(addr);
+ if (pdef.kind != PortKind::Ar) {
+ switch (pdef.clk_pol) {
+ case ClkPolKind::Posedge:
+ if (!clk_pol)
+ clk = mem.module->Not(NEW_ID, clk);
+ break;
+ case ClkPolKind::Negedge:
+ if (clk_pol)
+ clk = mem.module->Not(NEW_ID, clk);
+ break;
+ case ClkPolKind::Anyedge:
+ for (auto cell: cells)
+ cell->setParam(stringf("\\PORT_%s_CLK_POL", name), clk_pol);
+ }
+ for (auto cell: cells) {
+ cell->setPort(stringf("\\PORT_%s_CLK", name), clk);
+ if (pdef.clk_en)
+ cell->setPort(stringf("\\PORT_%s_CLK_EN", name), clk_en);
+ }
+ }
+
+ // Width determination.
+ if (pdef.width_tied) {
+ rd_wide_log2 = wr_wide_log2 = wide_log2;
+ }
+ int hw_wr_wide_log2 = cfg.base_width_log2;
+ for (int i = 0; i < wr_wide_log2; i++)
+ if (cfg.hard_wide_mask & (1 << i))
+ hw_wr_wide_log2++;
+ if (hw_wr_wide_log2 < pdef.min_wr_wide_log2)
+ hw_wr_wide_log2 = pdef.min_wr_wide_log2;
+ if (hw_wr_wide_log2 > pdef.max_wr_wide_log2)
+ hw_wr_wide_log2 = pdef.max_wr_wide_log2;
+ int hw_rd_wide_log2 = cfg.base_width_log2;
+ for (int i = 0; i < rd_wide_log2; i++)
+ if (cfg.hard_wide_mask & (1 << i))
+ hw_rd_wide_log2++;
+ if (hw_rd_wide_log2 < pdef.min_rd_wide_log2)
+ hw_rd_wide_log2 = pdef.min_rd_wide_log2;
+ if (hw_rd_wide_log2 > pdef.max_rd_wide_log2)
+ hw_rd_wide_log2 = pdef.max_rd_wide_log2;
+ if (pdef.width_tied) {
+ // For unused ports, pick max available width,
+ // in case narrow ports require disabling parity
+ // bits etc.
+ if (wpidx == -1 && rpidx == -1) {
+ hw_wr_wide_log2 = pdef.max_wr_wide_log2;
+ hw_rd_wide_log2 = pdef.max_rd_wide_log2;
+ }
+ } else {
+ if (wpidx == -1)
+ hw_wr_wide_log2 = pdef.max_wr_wide_log2;
+ if (rpidx == -1)
+ hw_rd_wide_log2 = pdef.max_rd_wide_log2;
+ }
+ if (cfg.def->width_mode == WidthMode::PerPort) {
+ for (auto cell: cells) {
+ if (pdef.width_tied) {
+ cell->setParam(stringf("\\PORT_%s_WIDTH", name), cfg.def->dbits[hw_wr_wide_log2]);
+ } else {
+ if (pdef.kind != PortKind::Ar && pdef.kind != PortKind::Sr)
+ cell->setParam(stringf("\\PORT_%s_WR_WIDTH", name), cfg.def->dbits[hw_wr_wide_log2]);
+ if (pdef.kind != PortKind::Sw)
+ cell->setParam(stringf("\\PORT_%s_RD_WIDTH", name), cfg.def->dbits[hw_rd_wide_log2]);
+ }
+ }
+ }
+
+ // Address determination.
+ SigSpec hw_addr;
+ for (int x: hw_addr_swizzle) {
+ if (x == -1 || x >= GetSize(addr))
+ hw_addr.append(State::S0);
+ else
+ hw_addr.append(addr[x]);
+ }
+ for (int i = 0; i < hw_wr_wide_log2 && i < hw_rd_wide_log2; i++)
+ hw_addr[i] = State::S0;
+ for (auto cell: cells)
+ cell->setPort(stringf("\\PORT_%s_ADDR", name), hw_addr);
+
+ // Write part.
+ if (pdef.kind != PortKind::Ar && pdef.kind != PortKind::Sr) {
+ int width = cfg.def->dbits[hw_wr_wide_log2];
+ int effective_byte = cfg.def->byte;
+ if (effective_byte == 0 || effective_byte > width)
+ effective_byte = width;
+ if (wpidx != -1) {
+ auto &wport = mem.wr_ports[wpidx];
+ Swizzle port_swz = gen_swizzle(mem, cfg, wport.wide_log2, hw_wr_wide_log2);
+ std::vector<SigSpec> big_wren = generate_demux(mem, wpidx, port_swz);
+ for (int rd = 0; rd < cfg.repl_d; rd++) {
+ auto cell = cells[rd];
+ SigSpec hw_wdata;
+ SigSpec hw_wren;
+ for (auto &bit : port_swz.bits[rd]) {
+ if (!bit.valid) {
+ hw_wdata.append(State::Sx);
+ } else {
+ hw_wdata.append(wport.data[bit.bit]);
+ }
+ }
+ for (int i = 0; i < GetSize(port_swz.bits[rd]); i += effective_byte) {
+ auto &bit = port_swz.bits[rd][i];
+ if (!bit.valid) {
+ hw_wren.append(State::S0);
+ } else {
+ hw_wren.append(big_wren[bit.mux_idx][bit.bit]);
+ }
+ }
+ cell->setPort(stringf("\\PORT_%s_WR_DATA", name), hw_wdata);
+ if (pdef.wrbe_separate) {
+ // TODO make some use of it
+ SigSpec en = mem.module->ReduceOr(NEW_ID, hw_wren);
+ cell->setPort(stringf("\\PORT_%s_WR_EN", name), en);
+ cell->setPort(stringf("\\PORT_%s_WR_BE", name), hw_wren);
+ if (cfg.def->width_mode != WidthMode::Single)
+ cell->setParam(stringf("\\PORT_%s_WR_BE_WIDTH", name), GetSize(hw_wren));
+ } else {
+ cell->setPort(stringf("\\PORT_%s_WR_EN", name), hw_wren);
+ if (cfg.def->byte != 0 && cfg.def->width_mode != WidthMode::Single)
+ cell->setParam(stringf("\\PORT_%s_WR_EN_WIDTH", name), GetSize(hw_wren));
+ }
+ }
+ } else {
+ for (auto cell: cells) {
+ cell->setPort(stringf("\\PORT_%s_WR_DATA", name), Const(State::Sx, width));
+ SigSpec hw_wren = Const(State::S0, width / effective_byte);
+ if (pdef.wrbe_separate) {
+ cell->setPort(stringf("\\PORT_%s_WR_EN", name), State::S0);
+ cell->setPort(stringf("\\PORT_%s_WR_BE", name), hw_wren);
+ cell->setParam(stringf("\\PORT_%s_WR_BE_WIDTH", name), GetSize(hw_wren));
+ } else {
+ cell->setPort(stringf("\\PORT_%s_WR_EN", name), hw_wren);
+ if (cfg.def->byte != 0)
+ cell->setParam(stringf("\\PORT_%s_WR_EN_WIDTH", name), GetSize(hw_wren));
+ }
+ }
+ }
+ }
+
+ // Read part.
+ if (pdef.kind != PortKind::Sw) {
+ int width = cfg.def->dbits[hw_rd_wide_log2];
+ if (rpidx != -1) {
+ auto &rport = mem.rd_ports[rpidx];
+ auto &rpcfg = cfg.rd_ports[rpidx];
+ Swizzle port_swz = gen_swizzle(mem, cfg, rport.wide_log2, hw_rd_wide_log2);
+ std::vector<SigSpec> big_rdata = generate_mux(mem, rpidx, port_swz);
+ for (int rd = 0; rd < cfg.repl_d; rd++) {
+ auto cell = cells[rd];
+ if (pdef.kind == PortKind::Sr || pdef.kind == PortKind::Srsw) {
+ if (pdef.rd_en)
+ cell->setPort(stringf("\\PORT_%s_RD_EN", name), rpcfg.rd_en_to_clk_en ? State::S1 : rport.en);
+ if (pdef.rdarstval != ResetValKind::None)
+ cell->setPort(stringf("\\PORT_%s_RD_ARST", name), rport.arst);
+ if (pdef.rdsrstval != ResetValKind::None)
+ cell->setPort(stringf("\\PORT_%s_RD_SRST", name), rport.srst);
+ if (pdef.rdinitval == ResetValKind::Any || pdef.rdinitval == ResetValKind::NoUndef) {
+ Const val = rport.init_value;
+ if (pdef.rdarstval == ResetValKind::Init && rport.arst != State::S0) {
+ log_assert(val.is_fully_undef() || val == rport.arst_value);
+ val = rport.arst_value;
+ }
+ if (pdef.rdsrstval == ResetValKind::Init && rport.srst != State::S0) {
+ log_assert(val.is_fully_undef() || val == rport.srst_value);
+ val = rport.srst_value;
+ }
+ std::vector<State> hw_val;
+ for (auto &bit : port_swz.bits[rd]) {
+ if (!bit.valid) {
+ hw_val.push_back(State::Sx);
+ } else {
+ hw_val.push_back(val.bits[bit.bit]);
+ }
+ }
+ if (pdef.rdinitval == ResetValKind::NoUndef)
+ clean_undef(hw_val);
+ cell->setParam(stringf("\\PORT_%s_RD_INIT_VALUE", name), hw_val);
+ }
+ if (pdef.rdarstval == ResetValKind::Any || pdef.rdarstval == ResetValKind::NoUndef) {
+ std::vector<State> hw_val;
+ for (auto &bit : port_swz.bits[rd]) {
+ if (!bit.valid) {
+ hw_val.push_back(State::Sx);
+ } else {
+ hw_val.push_back(rport.arst_value.bits[bit.bit]);
+ }
+ }
+ if (pdef.rdarstval == ResetValKind::NoUndef)
+ clean_undef(hw_val);
+ cell->setParam(stringf("\\PORT_%s_RD_ARST_VALUE", name), hw_val);
+ }
+ if (pdef.rdsrstval == ResetValKind::Any || pdef.rdsrstval == ResetValKind::NoUndef) {
+ std::vector<State> hw_val;
+ for (auto &bit : port_swz.bits[rd]) {
+ if (!bit.valid) {
+ hw_val.push_back(State::Sx);
+ } else {
+ hw_val.push_back(rport.srst_value.bits[bit.bit]);
+ }
+ }
+ if (pdef.rdsrstval == ResetValKind::NoUndef)
+ clean_undef(hw_val);
+ cell->setParam(stringf("\\PORT_%s_RD_SRST_VALUE", name), hw_val);
+ }
+ }
+ SigSpec hw_rdata = mem.module->addWire(NEW_ID, width);
+ cell->setPort(stringf("\\PORT_%s_RD_DATA", name), hw_rdata);
+ SigSpec lhs;
+ SigSpec rhs;
+ for (int i = 0; i < GetSize(hw_rdata); i++) {
+ auto &bit = port_swz.bits[rd][i];
+ if (bit.valid) {
+ lhs.append(big_rdata[bit.mux_idx][bit.bit]);
+ rhs.append(hw_rdata[i]);
+ }
+ }
+ mem.module->connect(lhs, rhs);
+ }
+ } else {
+ for (auto cell: cells) {
+ if (pdef.kind == PortKind::Sr || pdef.kind == PortKind::Srsw) {
+ if (pdef.rd_en)
+ cell->setPort(stringf("\\PORT_%s_RD_EN", name), State::S0);
+ if (pdef.rdarstval != ResetValKind::None)
+ cell->setPort(stringf("\\PORT_%s_RD_ARST", name), State::S0);
+ if (pdef.rdsrstval != ResetValKind::None)
+ cell->setPort(stringf("\\PORT_%s_RD_SRST", name), State::S0);
+ if (pdef.rdinitval == ResetValKind::Any)
+ cell->setParam(stringf("\\PORT_%s_RD_INIT_VALUE", name), Const(State::Sx, width));
+ else if (pdef.rdinitval == ResetValKind::NoUndef)
+ cell->setParam(stringf("\\PORT_%s_RD_INIT_VALUE", name), Const(State::S0, width));
+ if (pdef.rdarstval == ResetValKind::Any)
+ cell->setParam(stringf("\\PORT_%s_RD_ARST_VALUE", name), Const(State::Sx, width));
+ else if (pdef.rdarstval == ResetValKind::NoUndef)
+ cell->setParam(stringf("\\PORT_%s_RD_ARST_VALUE", name), Const(State::S0, width));
+ if (pdef.rdsrstval == ResetValKind::Any)
+ cell->setParam(stringf("\\PORT_%s_RD_SRST_VALUE", name), Const(State::Sx, width));
+ else if (pdef.rdsrstval == ResetValKind::NoUndef)
+ cell->setParam(stringf("\\PORT_%s_RD_SRST_VALUE", name), Const(State::S0, width));
+ }
+ SigSpec hw_rdata = mem.module->addWire(NEW_ID, width);
+ cell->setPort(stringf("\\PORT_%s_RD_DATA", name), hw_rdata);
+ }
+ }
+ }
+}
+
+void MemMapping::emit(const MemConfig &cfg) {
+ log("mapping memory %s.%s via %s\n", log_id(mem.module->name), log_id(mem.memid), log_id(cfg.def->id));
+ // First, handle emulations.
+ if (cfg.emu_read_first)
+ mem.emulate_read_first(&worker.initvals);
+ for (int pidx = 0; pidx < GetSize(mem.rd_ports); pidx++) {
+ auto &pcfg = cfg.rd_ports[pidx];
+ auto &port = mem.rd_ports[pidx];
+ if (pcfg.emu_sync)
+ mem.extract_rdff(pidx, &worker.initvals);
+ else if (pcfg.emu_en)
+ mem.emulate_rden(pidx, &worker.initvals);
+ else {
+ if (pcfg.emu_srst_en_prio) {
+ if (port.ce_over_srst)
+ mem.emulate_rd_ce_over_srst(pidx);
+ else
+ mem.emulate_rd_srst_over_ce(pidx);
+ }
+ mem.emulate_reset(pidx, pcfg.emu_init, pcfg.emu_arst, pcfg.emu_srst, &worker.initvals);
+ }
+ }
+ for (int pidx = 0; pidx < GetSize(mem.wr_ports); pidx++) {
+ auto &pcfg = cfg.wr_ports[pidx];
+ for (int opidx: pcfg.emu_prio) {
+ mem.emulate_priority(opidx, pidx, &worker.initvals);
+ }
+ }
+ for (int pidx = 0; pidx < GetSize(mem.rd_ports); pidx++) {
+ auto &port = mem.rd_ports[pidx];
+ auto &pcfg = cfg.rd_ports[pidx];
+ for (int opidx: pcfg.emu_trans) {
+ // The port may no longer be transparent due to transparency being
+ // nuked as part of emu_sync or emu_prio.
+ if (port.transparency_mask[opidx])
+ mem.emulate_transparency(opidx, pidx, &worker.initvals);
+ }
+ }
+
+ // tgt (repl, port group, port) -> mem (wr port, rd port), where -1 means no port.
+ std::vector<std::vector<std::vector<std::pair<int, int>>>> ports(cfg.repl_port);
+ for (int i = 0; i < cfg.repl_port; i++)
+ ports[i].resize(cfg.def->port_groups.size());
+ for (int i = 0; i < GetSize(cfg.wr_ports); i++) {
+ auto &pcfg = cfg.wr_ports[i];
+ for (int j = 0; j < cfg.repl_port; j++) {
+ if (j == 0) {
+ ports[j][pcfg.port_group].push_back({i, pcfg.rd_port});
+ } else {
+ ports[j][pcfg.port_group].push_back({i, -1});
+ }
+ }
+ }
+ for (int i = 0; i < GetSize(cfg.rd_ports); i++) {
+ auto &pcfg = cfg.rd_ports[i];
+ if (pcfg.wr_port != -1)
+ continue;
+ auto &pg = cfg.def->port_groups[pcfg.port_group];
+ int j = 0;
+ while (GetSize(ports[j][pcfg.port_group]) >= GetSize(pg.names))
+ j++;
+ ports[j][pcfg.port_group].push_back({-1, i});
+ }
+
+ Swizzle init_swz = gen_swizzle(mem, cfg, 0, GetSize(cfg.def->dbits) - 1);
+ Const init_data = mem.get_init_data();
+
+ std::vector<int> hw_addr_swizzle;
+ for (int i = 0; i < cfg.base_width_log2; i++)
+ hw_addr_swizzle.push_back(-1);
+ for (int i = 0; i < init_swz.addr_shift; i++)
+ if (!(cfg.emu_wide_mask & 1 << i))
+ hw_addr_swizzle.push_back(i);
+ log_assert(GetSize(hw_addr_swizzle) == cfg.def->abits);
+
+ for (int rp = 0; rp < cfg.repl_port; rp++) {
+ std::vector<Cell *> cells;
+ for (int rd = 0; rd < cfg.repl_d; rd++) {
+ Cell *cell = mem.module->addCell(stringf("%s.%d.%d", mem.memid.c_str(), rp, rd), cfg.def->id);
+ if (cfg.def->width_mode == WidthMode::Global)
+ cell->setParam(ID::WIDTH, cfg.def->dbits[cfg.base_width_log2]);
+ if (cfg.def->widthscale) {
+ std::vector<State> val;
+ for (auto &bit: init_swz.bits[rd])
+ val.push_back(bit.valid ? State::S1 : State::S0);
+ cell->setParam(ID::BITS_USED, val);
+ }
+ for (auto &it: cfg.def->options)
+ cell->setParam(stringf("\\OPTION_%s", it.first.c_str()), it.second);
+ for (int i = 0; i < GetSize(cfg.def->shared_clocks); i++) {
+ auto &cdef = cfg.def->shared_clocks[i];
+ auto &ccfg = cfg.shared_clocks[i];
+ if (cdef.anyedge) {
+ cell->setParam(stringf("\\CLK_%s_POL", cdef.name.c_str()), ccfg.used ? ccfg.polarity : true);
+ cell->setPort(stringf("\\CLK_%s", cdef.name.c_str()), ccfg.used ? ccfg.clk : State::S0);
+ } else {
+ SigSpec sig = ccfg.used ? ccfg.clk : State::S0;
+ if (ccfg.used && ccfg.invert)
+ sig = mem.module->Not(NEW_ID, sig);
+ cell->setPort(stringf("\\CLK_%s", cdef.name.c_str()), sig);
+ }
+ }
+ if (cfg.def->init == MemoryInitKind::Any || cfg.def->init == MemoryInitKind::NoUndef) {
+ std::vector<State> initval;
+ for (int hwa = 0; hwa < (1 << cfg.def->abits); hwa += 1 << (GetSize(cfg.def->dbits) - 1)) {
+ for (auto &bit: init_swz.bits[rd]) {
+ if (!bit.valid) {
+ initval.push_back(State::Sx);
+ } else {
+ int addr = bit.addr;
+ for (int i = GetSize(cfg.def->dbits) - 1; i < cfg.def->abits; i++)
+ if (hwa & 1 << i)
+ addr += 1 << hw_addr_swizzle[i];
+ if (addr >= mem.start_offset && addr < mem.start_offset + mem.size)
+ initval.push_back(init_data.bits[(addr - mem.start_offset) * mem.width + bit.bit]);
+ else
+ initval.push_back(State::Sx);
+ }
+ }
+ }
+ if (cfg.def->init == MemoryInitKind::NoUndef)
+ clean_undef(initval);
+ cell->setParam(ID::INIT, initval);
+ }
+ cells.push_back(cell);
+ }
+ for (int pgi = 0; pgi < GetSize(cfg.def->port_groups); pgi++) {
+ auto &pg = cfg.def->port_groups[pgi];
+ for (int pi = 0; pi < GetSize(pg.names); pi++) {
+ bool used = pi < GetSize(ports[rp][pgi]);
+ bool used_r = false;
+ bool used_w = false;
+ if (used) {
+ auto &pd = ports[rp][pgi][pi];
+ const PortVariant *pdef;
+ if (pd.first != -1)
+ pdef = cfg.wr_ports[pd.first].def;
+ else
+ pdef = cfg.rd_ports[pd.second].def;
+ used_w = pd.first != -1;
+ used_r = pd.second != -1;
+ emit_port(cfg, cells, *pdef, pg.names[pi].c_str(), pd.first, pd.second, hw_addr_swizzle);
+ } else {
+ emit_port(cfg, cells, pg.variants[0], pg.names[pi].c_str(), -1, -1, hw_addr_swizzle);
+ }
+ if (pg.optional)
+ for (auto cell: cells)
+ cell->setParam(stringf("\\PORT_%s_USED", pg.names[pi].c_str()), used);
+ if (pg.optional_rw)
+ for (auto cell: cells) {
+ cell->setParam(stringf("\\PORT_%s_RD_USED", pg.names[pi].c_str()), used_r);
+ cell->setParam(stringf("\\PORT_%s_WR_USED", pg.names[pi].c_str()), used_w);
+ }
+ }
+ }
+ }
+ mem.remove();
+}
+
+struct MemoryLibMapPass : public Pass {
+ MemoryLibMapPass() : Pass("memory_libmap", "map memories to cells") { }
+ void help() override
+ {
+ // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|
+ log("\n");
+ log(" memory_libmap -lib <library_file> [-D <condition>] [selection]\n");
+ log("\n");
+ log("This pass takes a description of available RAM cell types and maps\n");
+ log("all selected memories to one of them, or leaves them to be mapped to FFs.\n");
+ log("\n");
+ log(" -lib <library_file>\n");
+ log(" Selects a library file containing RAM cell definitions. This option\n");
+ log(" can be passed more than once to select multiple libraries.\n");
+ log(" See passes/memory/memlib.md for description of the library format.\n");
+ log("\n");
+ log(" -D <condition>\n");
+ log(" Enables a condition that can be checked within the library file\n");
+ log(" to eg. select between slightly different hardware variants.\n");
+ log(" This option can be passed any number of times.\n");
+ log("\n");
+ log(" -logic-cost-rom <num>\n");
+ log(" -logic-cost-ram <num>\n");
+ log(" Sets the cost of a single bit for memory lowered to soft logic.\n");
+ log("\n");
+ log(" -no-auto-distributed\n");
+ log(" -no-auto-block\n");
+ log(" -no-auto-huge\n");
+ log(" Disables automatic mapping of given kind of RAMs. Manual mapping\n");
+ log(" (using ram_style or other attributes) is still supported.\n");
+ log("\n");
+ }
+ void execute(std::vector<std::string> args, RTLIL::Design *design) override
+ {
+ std::vector<std::string> lib_files;
+ pool<std::string> defines;
+ PassOptions opts;
+ opts.no_auto_distributed = false;
+ opts.no_auto_block = false;
+ opts.no_auto_huge = false;
+ opts.logic_cost_ram = 1.0;
+ opts.logic_cost_rom = 1.0/16.0;
+ log_header(design, "Executing MEMORY_LIBMAP pass (mapping memories to cells).\n");
+
+ size_t argidx;
+ for (argidx = 1; argidx < args.size(); argidx++) {
+ if (args[argidx] == "-lib" && argidx+1 < args.size()) {
+ lib_files.push_back(args[++argidx]);
+ continue;
+ }
+ if (args[argidx] == "-D" && argidx+1 < args.size()) {
+ defines.insert(args[++argidx]);
+ continue;
+ }
+ if (args[argidx] == "-no-auto-distributed") {
+ opts.no_auto_distributed = true;
+ continue;
+ }
+ if (args[argidx] == "-no-auto-block") {
+ opts.no_auto_block = true;
+ continue;
+ }
+ if (args[argidx] == "-no-auto-huge") {
+ opts.no_auto_huge = true;
+ continue;
+ }
+ if (args[argidx] == "-logic-cost-rom" && argidx+1 < args.size()) {
+ opts.logic_cost_rom = strtod(args[++argidx].c_str(), nullptr);
+ continue;
+ }
+ if (args[argidx] == "-logic-cost-ram" && argidx+1 < args.size()) {
+ opts.logic_cost_ram = strtod(args[++argidx].c_str(), nullptr);
+ continue;
+ }
+ break;
+ }
+ extra_args(args, argidx, design);
+
+ Library lib = parse_library(lib_files, defines);
+
+ for (auto module : design->selected_modules()) {
+ MapWorker worker(module);
+ auto mems = Mem::get_selected_memories(module);
+ for (auto &mem : mems)
+ {
+ MemMapping map(worker, mem, lib, opts);
+ int idx = -1;
+ int best = map.logic_cost;
+ if (!map.logic_ok) {
+ if (map.cfgs.empty())
+ log_error("no valid mapping found for memory %s.%s\n", log_id(module->name), log_id(mem.memid));
+ idx = 0;
+ best = map.cfgs[0].cost;
+ }
+ for (int i = 0; i < GetSize(map.cfgs); i++) {
+ if (map.cfgs[i].cost < best) {
+ idx = i;
+ best = map.cfgs[i].cost;
+ }
+ }
+ if (idx == -1) {
+ log("using FF mapping for memory %s.%s\n", log_id(module->name), log_id(mem.memid));
+ } else {
+ map.emit(map.cfgs[idx]);
+ }
+ }
+ }
+ }
+} MemoryLibMapPass;
+
+PRIVATE_NAMESPACE_END
--
cgit v1.2.3