diff options
Diffstat (limited to 'passes/memory')
| -rw-r--r-- | passes/memory/Makefile.inc | 1 | ||||
| -rw-r--r-- | passes/memory/memory.cc | 30 | ||||
| -rw-r--r-- | passes/memory/memory_bram.cc | 560 | ||||
| -rw-r--r-- | passes/memory/memory_collect.cc | 234 | ||||
| -rw-r--r-- | passes/memory/memory_dff.cc | 778 | ||||
| -rw-r--r-- | passes/memory/memory_map.cc | 252 | ||||
| -rw-r--r-- | passes/memory/memory_memx.cc | 75 | ||||
| -rw-r--r-- | passes/memory/memory_narrow.cc | 67 | ||||
| -rw-r--r-- | passes/memory/memory_nordff.cc | 75 | ||||
| -rw-r--r-- | passes/memory/memory_share.cc | 959 | ||||
| -rw-r--r-- | passes/memory/memory_unpack.cc | 118 |
11 files changed, 1419 insertions, 1730 deletions
diff --git a/passes/memory/Makefile.inc b/passes/memory/Makefile.inc index e468c3a07..5a2c4ecfc 100644 --- a/passes/memory/Makefile.inc +++ b/passes/memory/Makefile.inc @@ -8,4 +8,5 @@ OBJS += passes/memory/memory_bram.o OBJS += passes/memory/memory_map.o OBJS += passes/memory/memory_memx.o OBJS += passes/memory/memory_nordff.o +OBJS += passes/memory/memory_narrow.o diff --git a/passes/memory/memory.cc b/passes/memory/memory.cc index 282517992..bac547c1a 100644 --- a/passes/memory/memory.cc +++ b/passes/memory/memory.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -31,16 +31,19 @@ struct MemoryPass : public Pass { { // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| log("\n"); - log(" memory [-nomap] [-nordff] [-memx] [-bram <bram_rules>] [selection]\n"); + log(" memory [-nomap] [-nordff] [-nowiden] [-nosat] [-memx] [-bram <bram_rules>] [selection]\n"); log("\n"); log("This pass calls all the other memory_* passes in a useful order:\n"); log("\n"); log(" opt_mem\n"); - log(" memory_dff [-nordff] (-memx implies -nordff)\n"); - log(" opt_clean\n"); - log(" memory_share\n"); + log(" opt_mem_priority\n"); + log(" opt_mem_feedback\n"); + log(" memory_dff (skipped if called with -nordff or -memx)\n"); log(" opt_clean\n"); + log(" memory_share [-nowiden] [-nosat]\n"); + log(" opt_mem_widen\n"); log(" memory_memx (when called with -memx)\n"); + log(" opt_clean\n"); log(" memory_collect\n"); log(" memory_bram -rules <bram_rules> (when called with -bram)\n"); log(" memory_map (skipped if called with -nomap)\n"); @@ -55,6 +58,7 @@ struct MemoryPass : public Pass { bool flag_nordff = false; bool flag_memx = false; string memory_bram_opts; + string memory_share_opts; log_header(design, "Executing MEMORY pass.\n"); log_push(); @@ -74,6 +78,14 @@ struct MemoryPass : public Pass { flag_memx = true; continue; } + if (args[argidx] == "-nowiden") { + memory_share_opts += " -nowiden"; + continue; + } + if (args[argidx] == "-nosat") { + memory_share_opts += " -nosat"; + continue; + } if (argidx+1 < args.size() && args[argidx] == "-bram") { memory_bram_opts += " -rules " + args[++argidx]; continue; @@ -83,9 +95,13 @@ struct MemoryPass : public Pass { extra_args(args, argidx, design); Pass::call(design, "opt_mem"); - Pass::call(design, flag_nordff ? "memory_dff -nordff" : "memory_dff"); + Pass::call(design, "opt_mem_priority"); + Pass::call(design, "opt_mem_feedback"); + if (!flag_nordff) + Pass::call(design, "memory_dff"); Pass::call(design, "opt_clean"); - Pass::call(design, "memory_share"); + Pass::call(design, "memory_share" + memory_share_opts); + Pass::call(design, "opt_mem_widen"); if (flag_memx) Pass::call(design, "memory_memx"); Pass::call(design, "opt_clean"); diff --git a/passes/memory/memory_bram.cc b/passes/memory/memory_bram.cc index 3cb0728b7..b1f45d5fc 100644 --- a/passes/memory/memory_bram.cc +++ b/passes/memory/memory_bram.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,6 +18,8 @@ */ #include "kernel/yosys.h" +#include "kernel/mem.h" +#include "kernel/ffinit.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN @@ -28,9 +30,6 @@ struct rules_t int group, index, dupidx; int wrmode, enable, transp, clocks, clkpol; - SigBit sig_clock; - SigSpec sig_addr, sig_data, sig_en; - bool effective_clkpol; bool make_transp; bool make_outreg; int mapped_port; @@ -91,7 +90,6 @@ struct rules_t pi.mapped_port = -1; pi.make_transp = false; pi.make_outreg = false; - pi.effective_clkpol = false; portinfos.push_back(pi); } return portinfos; @@ -400,17 +398,13 @@ struct rules_t } }; -bool replace_cell(Cell *cell, const rules_t &rules, const rules_t::bram_t &bram, const rules_t::match_t &match, dict<string, int> &match_properties, int mode) +bool replace_memory(Mem &mem, const rules_t &rules, FfInitVals *initvals, const rules_t::bram_t &bram, const rules_t::match_t &match, dict<string, int> &match_properties, int mode) { - Module *module = cell->module; + Module *module = mem.module; auto portinfos = bram.make_portinfos(); int dup_count = 1; - pair<SigBit, bool> make_transp_clk; - bool enable_make_transp = false; - int make_transp_enbits = 0; - dict<int, pair<SigBit, bool>> clock_domains; dict<int, bool> clock_polarities; dict<int, bool> read_transp; @@ -437,124 +431,33 @@ bool replace_cell(Cell *cell, const rules_t &rules, const rules_t::bram_t &bram, log(" Mapping to bram type %s (variant %d):\n", log_id(bram.name), bram.variant); // bram.dump_config(); - int mem_size = cell->getParam(ID::SIZE).as_int(); - int mem_abits = cell->getParam(ID::ABITS).as_int(); - int mem_width = cell->getParam(ID::WIDTH).as_int(); - // int mem_offset = cell->getParam(ID::OFFSET).as_int(); - - bool cell_init = !SigSpec(cell->getParam(ID::INIT)).is_fully_undef(); - vector<Const> initdata; - - if (cell_init) { - Const initparam = cell->getParam(ID::INIT); - initdata.reserve(mem_size); - for (int i=0; i < mem_size; i++) - initdata.push_back(initparam.extract(mem_width*i, mem_width, State::Sx)); - } - - int wr_ports = cell->getParam(ID::WR_PORTS).as_int(); - auto wr_clken = SigSpec(cell->getParam(ID::WR_CLK_ENABLE)); - auto wr_clkpol = SigSpec(cell->getParam(ID::WR_CLK_POLARITY)); - wr_clken.extend_u0(wr_ports); - wr_clkpol.extend_u0(wr_ports); - - SigSpec wr_en = cell->getPort(ID::WR_EN); - SigSpec wr_clk = cell->getPort(ID::WR_CLK); - SigSpec wr_data = cell->getPort(ID::WR_DATA); - SigSpec wr_addr = cell->getPort(ID::WR_ADDR); - - int rd_ports = cell->getParam(ID::RD_PORTS).as_int(); - auto rd_clken = SigSpec(cell->getParam(ID::RD_CLK_ENABLE)); - auto rd_clkpol = SigSpec(cell->getParam(ID::RD_CLK_POLARITY)); - auto rd_transp = SigSpec(cell->getParam(ID::RD_TRANSPARENT)); - rd_clken.extend_u0(rd_ports); - rd_clkpol.extend_u0(rd_ports); - rd_transp.extend_u0(rd_ports); - - SigSpec rd_en = cell->getPort(ID::RD_EN); - SigSpec rd_clk = cell->getPort(ID::RD_CLK); - SigSpec rd_data = cell->getPort(ID::RD_DATA); - SigSpec rd_addr = cell->getPort(ID::RD_ADDR); - - if (match.shuffle_enable && bram.dbits >= portinfos.at(match.shuffle_enable - 'A').enable*2 && portinfos.at(match.shuffle_enable - 'A').enable > 0 && wr_ports > 0) + std::vector<int> shuffle_map; + if (match.shuffle_enable && bram.dbits >= portinfos.at(match.shuffle_enable - 'A').enable*2 && portinfos.at(match.shuffle_enable - 'A').enable > 0 && !mem.wr_ports.empty()) { int bucket_size = bram.dbits / portinfos.at(match.shuffle_enable - 'A').enable; log(" Shuffle bit order to accommodate enable buckets of size %d..\n", bucket_size); - // extract unshuffled data/enable bits - - std::vector<SigSpec> old_wr_en; - std::vector<SigSpec> old_wr_data; - std::vector<SigSpec> old_rd_data; - - for (int i = 0; i < wr_ports; i++) { - old_wr_en.push_back(wr_en.extract(i*mem_width, mem_width)); - old_wr_data.push_back(wr_data.extract(i*mem_width, mem_width)); - } - - for (int i = 0; i < rd_ports; i++) - old_rd_data.push_back(rd_data.extract(i*mem_width, mem_width)); - // analyze enable structure std::vector<SigSpec> en_order; dict<SigSpec, vector<int>> bits_wr_en; - for (int i = 0; i < mem_width; i++) { + for (int i = 0; i < mem.width; i++) { SigSpec sig; - for (int j = 0; j < wr_ports; j++) - sig.append(old_wr_en[j][i]); + for (auto &port : mem.wr_ports) + sig.append(port.en[i]); if (bits_wr_en.count(sig) == 0) en_order.push_back(sig); bits_wr_en[sig].push_back(i); } - // re-create memory ports - - std::vector<SigSpec> new_wr_en(GetSize(old_wr_en)); - std::vector<SigSpec> new_wr_data(GetSize(old_wr_data)); - std::vector<SigSpec> new_rd_data(GetSize(old_rd_data)); - std::vector<std::vector<State>> new_initdata; - std::vector<int> shuffle_map; - - if (cell_init) - new_initdata.resize(mem_size); - for (auto &it : en_order) { - auto &bits = bits_wr_en.at(it); - int buckets = (GetSize(bits) + bucket_size - 1) / bucket_size; - int fillbits = buckets*bucket_size - GetSize(bits); - SigBit fillbit; - - for (int i = 0; i < GetSize(bits); i++) { - for (int j = 0; j < wr_ports; j++) { - new_wr_en[j].append(old_wr_en[j][bits[i]]); - new_wr_data[j].append(old_wr_data[j][bits[i]]); - fillbit = old_wr_en[j][bits[i]]; - } - for (int j = 0; j < rd_ports; j++) - new_rd_data[j].append(old_rd_data[j][bits[i]]); - if (cell_init) { - for (int j = 0; j < mem_size; j++) - new_initdata[j].push_back(initdata[j][bits[i]]); - } - shuffle_map.push_back(bits[i]); - } + for (auto bit : bits_wr_en.at(it)) + shuffle_map.push_back(bit); - for (int i = 0; i < fillbits; i++) { - for (int j = 0; j < wr_ports; j++) { - new_wr_en[j].append(fillbit); - new_wr_data[j].append(State::S0); - } - for (int j = 0; j < rd_ports; j++) - new_rd_data[j].append(State::Sx); - if (cell_init) { - for (int j = 0; j < mem_size; j++) - new_initdata[j].push_back(State::Sx); - } + while (GetSize(shuffle_map) % bucket_size) shuffle_map.push_back(-1); - } } log(" Results of bit order shuffling:"); @@ -563,53 +466,38 @@ bool replace_cell(Cell *cell, const rules_t &rules, const rules_t::bram_t &bram, log("\n"); // update mem_*, wr_*, and rd_* variables - - mem_width = GetSize(new_wr_en.front()); - wr_en = SigSpec(0, wr_ports * mem_width); - wr_data = SigSpec(0, wr_ports * mem_width); - rd_data = SigSpec(0, rd_ports * mem_width); - - for (int i = 0; i < wr_ports; i++) { - wr_en.replace(i*mem_width, new_wr_en[i]); - wr_data.replace(i*mem_width, new_wr_data[i]); - } - - for (int i = 0; i < rd_ports; i++) - rd_data.replace(i*mem_width, new_rd_data[i]); - - if (cell_init) { - for (int i = 0; i < mem_size; i++) - initdata[i] = Const(new_initdata[i]); - } + } else { + for (int i = 0; i < mem.width; i++) + shuffle_map.push_back(i); } + // Align width up to dbits. + while (GetSize(shuffle_map) % bram.dbits) + shuffle_map.push_back(-1); + // assign write ports pair<SigBit, bool> wr_clkdom; - for (int cell_port_i = 0, bram_port_i = 0; cell_port_i < wr_ports; cell_port_i++) + for (int cell_port_i = 0, bram_port_i = 0; cell_port_i < GetSize(mem.wr_ports); cell_port_i++) { - bool clken = wr_clken[cell_port_i] == State::S1; - bool clkpol = wr_clkpol[cell_port_i] == State::S1; - SigBit clksig = wr_clk[cell_port_i]; + auto &port = mem.wr_ports[cell_port_i]; - pair<SigBit, bool> clkdom(clksig, clkpol); - if (!clken) + pair<SigBit, bool> clkdom(port.clk, port.clk_polarity); + if (!port.clk_enable) clkdom = pair<SigBit, bool>(State::S1, false); wr_clkdom = clkdom; log(" Write port #%d is in clock domain %s%s.\n", cell_port_i, clkdom.second ? "" : "!", - clken ? log_signal(clkdom.first) : "~async~"); + port.clk_enable ? log_signal(clkdom.first) : "~async~"); for (; bram_port_i < GetSize(portinfos); bram_port_i++) { auto &pi = portinfos[bram_port_i]; - make_transp_enbits = pi.enable; - make_transp_clk = clkdom; if (pi.wrmode != 1) skip_bram_wport: continue; - if (clken) { + if (port.clk_enable) { if (pi.clocks == 0) { log(" Bram port %c%d has incompatible clock type.\n", pi.group + 'A', pi.index + 1); goto skip_bram_wport; @@ -618,7 +506,7 @@ bool replace_cell(Cell *cell, const rules_t &rules, const rules_t::bram_t &bram, log(" Bram port %c%d is in a different clock domain.\n", pi.group + 'A', pi.index + 1); goto skip_bram_wport; } - if (clock_polarities.count(pi.clkpol) && clock_polarities.at(pi.clkpol) != clkpol) { + if (clock_polarities.count(pi.clkpol) && clock_polarities.at(pi.clkpol) != port.clk_polarity) { log(" Bram port %c%d has incompatible clock polarity.\n", pi.group + 'A', pi.index + 1); goto skip_bram_wport; } @@ -629,33 +517,36 @@ bool replace_cell(Cell *cell, const rules_t &rules, const rules_t::bram_t &bram, } } - SigSpec sig_en; - SigBit last_en_bit = State::S1; - for (int i = 0; i < mem_width; i++) { - if (pi.enable && i % (bram.dbits / pi.enable) == 0) { - last_en_bit = wr_en[i + cell_port_i*mem_width]; - sig_en.append(last_en_bit); - } - if (last_en_bit != wr_en[i + cell_port_i*mem_width]) { - log(" Bram port %c%d has incompatible enable structure.\n", pi.group + 'A', pi.index + 1); - goto skip_bram_wport; + // We need to check enable compatibility of this port, as well as all + // ports that have priority over this one (because they will be involved + // in emulate_priority logic). + + for (int i = 0; i < GetSize(mem.wr_ports); i++) { + auto &oport = mem.wr_ports[i]; + if (i != cell_port_i && !oport.priority_mask[cell_port_i]) + continue; + SigBit last_en_bit = State::S1; + for (int j = 0; j < GetSize(shuffle_map); j++) { + if (shuffle_map[j] == -1) + continue; + SigBit en_bit = oport.en[shuffle_map[j]]; + if (pi.enable && j % (bram.dbits / pi.enable) == 0) + last_en_bit = en_bit; + if (last_en_bit != en_bit) { + log(" Bram port %c%d has incompatible enable structure.\n", pi.group + 'A', pi.index + 1); + goto skip_bram_wport; + } } } log(" Mapped to bram port %c%d.\n", pi.group + 'A', pi.index + 1); pi.mapped_port = cell_port_i; - if (clken) { + if (port.clk_enable) { clock_domains[pi.clocks] = clkdom; clock_polarities[pi.clkpol] = clkdom.second; - pi.sig_clock = clkdom.first; - pi.effective_clkpol = clkdom.second; } - pi.sig_en = sig_en; - pi.sig_addr = wr_addr.extract(cell_port_i*mem_abits, mem_abits); - pi.sig_data = wr_data.extract(cell_port_i*mem_width, mem_width); - bram_port_i++; goto mapped_wr_port; } @@ -678,10 +569,6 @@ grow_read_ports:; for (auto &pi : portinfos) { if (pi.wrmode == 0) { pi.mapped_port = -1; - pi.sig_clock = SigBit(); - pi.sig_addr = SigSpec(); - pi.sig_data = SigSpec(); - pi.sig_en = SigSpec(); pi.make_outreg = false; pi.make_transp = false; } @@ -710,23 +597,27 @@ grow_read_ports:; // assign read ports - for (int cell_port_i = 0; cell_port_i < rd_ports; cell_port_i++) + for (int cell_port_i = 0; cell_port_i < GetSize(mem.rd_ports); cell_port_i++) { - bool clken = rd_clken[cell_port_i] == State::S1; - bool clkpol = rd_clkpol[cell_port_i] == State::S1; - bool transp = rd_transp[cell_port_i] == State::S1; - SigBit clksig = rd_clk[cell_port_i]; - - if (wr_ports == 0) - transp = false; + auto &port = mem.rd_ports[cell_port_i]; + bool transp = false; + bool non_transp = false; + + if (port.clk_enable) { + for (int i = 0; i < GetSize(mem.wr_ports); i++) + if (port.transparency_mask[i]) + transp = true; + else if (!port.collision_x_mask[i]) + non_transp = true; + } - pair<SigBit, bool> clkdom(clksig, clkpol); - if (!clken) + pair<SigBit, bool> clkdom(port.clk, port.clk_polarity); + if (!port.clk_enable) clkdom = pair<SigBit, bool>(State::S1, false); log(" Read port #%d is in clock domain %s%s.\n", cell_port_i, clkdom.second ? "" : "!", - clken ? log_signal(clkdom.first) : "~async~"); + port.clk_enable ? log_signal(clkdom.first) : "~async~"); for (int bram_port_i = 0; bram_port_i < GetSize(portinfos); bram_port_i++) { @@ -736,7 +627,7 @@ grow_read_ports:; skip_bram_rport: continue; - if (clken) { + if (port.clk_enable) { if (pi.clocks == 0) { if (match.make_outreg) { pi.make_outreg = true; @@ -749,25 +640,17 @@ grow_read_ports:; log(" Bram port %c%d.%d is in a different clock domain.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); goto skip_bram_rport; } - if (clock_polarities.count(pi.clkpol) && clock_polarities.at(pi.clkpol) != clkpol) { + if (clock_polarities.count(pi.clkpol) && clock_polarities.at(pi.clkpol) != port.clk_polarity) { log(" Bram port %c%d.%d has incompatible clock polarity.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); goto skip_bram_rport; } - if (rd_en[cell_port_i] != State::S1 && pi.enable == 0) { - log(" Bram port %c%d.%d has no read enable input.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); + if (non_transp && read_transp.count(pi.transp) && read_transp.at(pi.transp)) { + log(" Bram port %c%d.%d has incompatible read transparency.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); goto skip_bram_rport; } - skip_bram_rport_clkcheck: - if (read_transp.count(pi.transp) && read_transp.at(pi.transp) != transp) { - if (match.make_transp && wr_ports <= 1) { + if (transp && (non_transp || (read_transp.count(pi.transp) && !read_transp.at(pi.transp)))) { + if (match.make_transp) { pi.make_transp = true; - if (pi.clocks != 0) { - if (wr_ports == 1 && wr_clkdom != clkdom) { - log(" Bram port %c%d.%d cannot have soft transparency logic added as read and write clock domains differ.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); - goto skip_bram_rport; - } - enable_make_transp = true; - } } else { log(" Bram port %c%d.%d has incompatible read transparency.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); goto skip_bram_rport; @@ -780,22 +663,19 @@ grow_read_ports:; } } + skip_bram_rport_clkcheck: log(" Mapped to bram port %c%d.%d.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); pi.mapped_port = cell_port_i; - if (clken) { + if (pi.clocks) { clock_domains[pi.clocks] = clkdom; clock_polarities[pi.clkpol] = clkdom.second; - if (!pi.make_transp) - read_transp[pi.transp] = transp; - pi.sig_clock = clkdom.first; - pi.sig_en = rd_en[cell_port_i]; - pi.effective_clkpol = clkdom.second; + if (non_transp) + read_transp[pi.transp] = false; + if (transp && !pi.make_transp) + read_transp[pi.transp] = true; } - pi.sig_addr = rd_addr.extract(cell_port_i*mem_abits, mem_abits); - pi.sig_data = rd_data.extract(cell_port_i*mem_width, mem_width); - if (grow_read_ports_cursor < cell_port_i) { grow_read_ports_cursor = cell_port_i; try_growing_more_read_ports = true; @@ -815,17 +695,19 @@ grow_read_ports:; // update properties and re-check conditions + int dcells = GetSize(shuffle_map) / bram.dbits; + int acells = (mem.size + (1 << bram.abits) - 1) / (1 << bram.abits); if (mode <= 1) { match_properties["dups"] = dup_count; match_properties["waste"] = match_properties["dups"] * match_properties["bwaste"]; - int cells = ((mem_width + bram.dbits - 1) / bram.dbits) * ((mem_size + (1 << bram.abits) - 1) / (1 << bram.abits)); + int cells = dcells * acells; match_properties["efficiency"] = (100 * match_properties["bits"]) / (dup_count * cells * bram.dbits * (1 << bram.abits)); - match_properties["dcells"] = ((mem_width + bram.dbits - 1) / bram.dbits); - match_properties["acells"] = ((mem_size + (1 << bram.abits) - 1) / (1 << bram.abits)); - match_properties["cells"] = match_properties["dcells"] * match_properties["acells"] * match_properties["dups"]; + match_properties["dcells"] = dcells; + match_properties["acells"] = acells; + match_properties["cells"] = cells * dup_count; log(" Updated properties: dups=%d waste=%d efficiency=%d\n", match_properties["dups"], match_properties["waste"], match_properties["efficiency"]); @@ -857,8 +739,8 @@ grow_read_ports:; bool exists = std::get<0>(term); IdString key = std::get<1>(term); const Const &value = std::get<2>(term); - auto it = cell->attributes.find(key); - if (it == cell->attributes.end()) { + auto it = mem.attributes.find(key); + if (it == mem.attributes.end()) { if (exists) continue; found = true; @@ -892,6 +774,90 @@ grow_read_ports:; return true; } + // At this point we are commited to replacing the RAM, and can mutate mem. + + // Apply make_outreg and make_transp where necessary. + for (auto &pi : portinfos) { + if (pi.mapped_port == -1 || pi.wrmode) + continue; + auto &port = mem.rd_ports[pi.mapped_port]; + if (pi.make_outreg) { + mem.extract_rdff(pi.mapped_port, initvals); + } else if (port.clk_enable) { + if (!pi.enable && port.en != State::S1) + mem.emulate_rden(pi.mapped_port, initvals); + else + mem.emulate_reset(pi.mapped_port, true, true, true, initvals); + } + if (pi.make_transp) { + for (int i = 0; i < GetSize(mem.wr_ports); i++) + if (port.transparency_mask[i]) + mem.emulate_transparency(i, pi.mapped_port, initvals); + } + } + + // We don't really support priorities, emulate them. + for (int i = 0; i < GetSize(mem.wr_ports); i++) + for (int j = 0; j < i; j++) + mem.emulate_priority(j, i, initvals); + + // Swizzle the init data. Do this before changing mem.width, so that get_init_data works. + bool cell_init = !mem.inits.empty(); + vector<Const> initdata; + if (cell_init) { + Const initparam = mem.get_init_data(); + initdata.reserve(mem.size); + for (int i = 0; i < mem.size; i++) { + std::vector<State> val; + for (auto idx : shuffle_map) { + if (idx == -1) + val.push_back(State::Sx); + else + val.push_back(initparam[mem.width * i + idx]); + } + initdata.push_back(Const(val)); + } + } + + // Now the big swizzle. + mem.width = GetSize(shuffle_map); + + // Swizzle write ports. + for (auto &port : mem.wr_ports) { + SigSpec new_en, new_data; + SigBit en_bit = State::S1; + for (auto idx : shuffle_map) { + if (idx == -1) { + new_data.append(State::Sx); + } else { + new_data.append(port.data[idx]); + en_bit = port.en[idx]; + } + new_en.append(en_bit); + } + port.en = new_en; + port.data = new_data; + } + + // Swizzle read ports. + for (auto &port : mem.rd_ports) { + SigSpec new_data = module->addWire(NEW_ID, mem.width); + Const new_init_value = Const(State::Sx, mem.width); + Const new_arst_value = Const(State::Sx, mem.width); + Const new_srst_value = Const(State::Sx, mem.width); + for (int i = 0; i < mem.width; i++) + if (shuffle_map[i] != -1) { + module->connect(port.data[shuffle_map[i]], new_data[i]); + new_init_value[i] = port.init_value[shuffle_map[i]]; + new_arst_value[i] = port.arst_value[shuffle_map[i]]; + new_srst_value[i] = port.srst_value[shuffle_map[i]]; + } + port.data = new_data; + port.init_value = new_init_value; + port.arst_value = new_arst_value; + port.srst_value = new_srst_value; + } + // prepare variant parameters dict<IdString, Const> variant_params; @@ -902,42 +868,28 @@ grow_read_ports:; dict<SigSpec, pair<SigSpec, SigSpec>> dout_cache; - for (int grid_d = 0; grid_d*bram.dbits < mem_width; grid_d++) + for (int grid_d = 0; grid_d < dcells; grid_d++) { - SigSpec mktr_wraddr, mktr_wrdata, mktr_wrdata_q; - vector<SigSpec> mktr_wren; - - if (enable_make_transp) { - mktr_wraddr = module->addWire(NEW_ID, bram.abits); - mktr_wrdata = module->addWire(NEW_ID, bram.dbits); - mktr_wrdata_q = module->addWire(NEW_ID, bram.dbits); - module->addDff(NEW_ID, make_transp_clk.first, mktr_wrdata, mktr_wrdata_q, make_transp_clk.second); - for (int grid_a = 0; grid_a*(1 << bram.abits) < mem_size; grid_a++) - mktr_wren.push_back(module->addWire(NEW_ID, make_transp_enbits)); - } - - for (int grid_a = 0; grid_a*(1 << bram.abits) < mem_size; grid_a++) + for (int grid_a = 0; grid_a < acells; grid_a++) for (int dupidx = 0; dupidx < dup_count; dupidx++) { - Cell *c = module->addCell(module->uniquify(stringf("%s.%d.%d.%d", cell->name.c_str(), grid_d, grid_a, dupidx)), bram.name); + Cell *c = module->addCell(module->uniquify(stringf("%s.%d.%d.%d", mem.memid.c_str(), grid_d, grid_a, dupidx)), bram.name); log(" Creating %s cell at grid position <%d %d %d>: %s\n", log_id(bram.name), grid_d, grid_a, dupidx, log_id(c)); for (auto &vp : variant_params) c->setParam(vp.first, vp.second); if (cell_init) { - int init_offset = grid_a*(1 << bram.abits); + int init_offset = grid_a*(1 << bram.abits) - mem.start_offset; int init_shift = grid_d*bram.dbits; int init_size = (1 << bram.abits); Const initparam(State::Sx, init_size*bram.dbits); - for (int i = 0; i < init_size; i++) { - State padding = State::Sx; + for (int i = 0; i < init_size; i++) for (int j = 0; j < bram.dbits; j++) - if (init_offset+i < GetSize(initdata) && init_shift+j < GetSize(initdata[init_offset+i])) + if (init_offset+i < GetSize(initdata) && init_offset+i >= 0) initparam[i*bram.dbits+j] = initdata[init_offset+i][init_shift+j]; else - initparam[i*bram.dbits+j] = padding; - } + initparam[i*bram.dbits+j] = State::Sx; c->setParam(ID::INIT, initparam); } @@ -949,101 +901,84 @@ grow_read_ports:; string prefix = stringf("%c%d", pi.group + 'A', pi.index + 1); const char *pf = prefix.c_str(); - if (pi.clocks && (!c->hasPort(stringf("\\CLK%d", (pi.clocks-1) % clocks_max + 1)) || pi.sig_clock.wire)) { - c->setPort(stringf("\\CLK%d", (pi.clocks-1) % clocks_max + 1), pi.sig_clock); - if (pi.clkpol > 1 && pi.sig_clock.wire) - c->setParam(stringf("\\CLKPOL%d", (pi.clkpol-1) % clkpol_max + 1), clock_polarities.at(pi.clkpol)); - if (pi.transp > 1 && pi.sig_clock.wire) - c->setParam(stringf("\\TRANSP%d", (pi.transp-1) % transp_max + 1), read_transp.at(pi.transp)); - } + if (pi.clocks && clock_domains.count(pi.clocks)) + c->setPort(stringf("\\CLK%d", (pi.clocks-1) % clocks_max + 1), clock_domains.at(pi.clocks).first); + if (pi.clkpol > 1 && clock_polarities.count(pi.clkpol)) + c->setParam(stringf("\\CLKPOL%d", (pi.clkpol-1) % clkpol_max + 1), clock_polarities.at(pi.clkpol)); + if (pi.transp > 1 && read_transp.count(pi.transp)) + c->setParam(stringf("\\TRANSP%d", (pi.transp-1) % transp_max + 1), read_transp.at(pi.transp)); SigSpec addr_ok; - if (GetSize(pi.sig_addr) > bram.abits) { - SigSpec extra_addr = pi.sig_addr.extract(bram.abits, GetSize(pi.sig_addr) - bram.abits); + SigSpec sig_addr; + if (pi.mapped_port >= 0) { + if (pi.wrmode == 1) + sig_addr = mem.wr_ports[pi.mapped_port].addr; + else + sig_addr = mem.rd_ports[pi.mapped_port].addr; + } + + if (GetSize(sig_addr) > bram.abits) { + SigSpec extra_addr = sig_addr.extract(bram.abits, GetSize(sig_addr) - bram.abits); SigSpec extra_addr_sel = SigSpec(grid_a, GetSize(extra_addr)); addr_ok = module->Eq(NEW_ID, extra_addr, extra_addr_sel); } - if (pi.enable) - { - SigSpec sig_en = pi.sig_en; + sig_addr.extend_u0(bram.abits); + c->setPort(stringf("\\%sADDR", pf), sig_addr); - if (pi.wrmode == 1) { - sig_en.extend_u0((grid_d+1) * pi.enable); - sig_en = sig_en.extract(grid_d * pi.enable, pi.enable); + if (pi.wrmode == 1) { + if (pi.mapped_port == -1) + { + if (pi.enable) + c->setPort(stringf("\\%sEN", pf), Const(State::S0, pi.enable)); + continue; } - if (!addr_ok.empty()) - sig_en = module->Mux(NEW_ID, SigSpec(0, GetSize(sig_en)), sig_en, addr_ok); - - c->setPort(stringf("\\%sEN", pf), sig_en); - - if (pi.wrmode == 1 && enable_make_transp) - module->connect(mktr_wren[grid_a], sig_en); - } + auto &port = mem.wr_ports[pi.mapped_port]; + SigSpec sig_data = port.data.extract(grid_d * bram.dbits, bram.dbits); + c->setPort(stringf("\\%sDATA", pf), sig_data); - SigSpec sig_addr = pi.sig_addr; - sig_addr.extend_u0(bram.abits); - c->setPort(stringf("\\%sADDR", pf), sig_addr); + if (pi.enable) + { + SigSpec sig_en; + int stride = bram.dbits / pi.enable; + for (int i = 0; i < pi.enable; i++) + sig_en.append(port.en[stride * i + grid_d * bram.dbits]); - if (pi.wrmode == 1 && enable_make_transp && grid_a == 0) - module->connect(mktr_wraddr, sig_addr); + if (!addr_ok.empty()) + sig_en = module->Mux(NEW_ID, SigSpec(0, GetSize(sig_en)), sig_en, addr_ok); - SigSpec sig_data = pi.sig_data; - sig_data.extend_u0((grid_d+1) * bram.dbits); - sig_data = sig_data.extract(grid_d * bram.dbits, bram.dbits); + c->setPort(stringf("\\%sEN", pf), sig_en); - if (pi.wrmode == 1) { - c->setPort(stringf("\\%sDATA", pf), sig_data); - if (enable_make_transp && grid_a == 0) - module->connect(mktr_wrdata, sig_data); + } } else { - SigSpec bram_dout = module->addWire(NEW_ID, bram.dbits); - c->setPort(stringf("\\%sDATA", pf), bram_dout); - if (pi.make_outreg && pi.make_transp) { - log(" Moving output register to address for transparent port %c%d.%d.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); - SigSpec sig_addr_q = module->addWire(NEW_ID, bram.abits); - module->addDff(NEW_ID, pi.sig_clock, sig_addr, sig_addr_q, pi.effective_clkpol); - c->setPort(stringf("\\%sADDR", pf), sig_addr_q); - } else if (pi.make_outreg) { - SigSpec bram_dout_q = module->addWire(NEW_ID, bram.dbits); - if (!pi.sig_en.empty()) - bram_dout = module->Mux(NEW_ID, bram_dout_q, bram_dout, pi.sig_en); - module->addDff(NEW_ID, pi.sig_clock, bram_dout, bram_dout_q, pi.effective_clkpol); - bram_dout = bram_dout_q; - } else if (pi.make_transp) { - log(" Adding extra logic for transparent port %c%d.%d.\n", pi.group + 'A', pi.index + 1, pi.dupidx + 1); - - SigSpec transp_en_d = module->Mux(NEW_ID, SigSpec(0, make_transp_enbits), - mktr_wren[grid_a], module->Eq(NEW_ID, mktr_wraddr, sig_addr)); - - SigSpec transp_en_q = module->addWire(NEW_ID, make_transp_enbits); - module->addDff(NEW_ID, make_transp_clk.first, transp_en_d, transp_en_q, make_transp_clk.second); - - for (int i = 0; i < make_transp_enbits; i++) { - int en_width = bram.dbits / make_transp_enbits; - SigSpec orig_bram_dout = bram_dout.extract(i * en_width, en_width); - SigSpec bypass_dout = mktr_wrdata_q.extract(i * en_width, en_width); - bram_dout.replace(i * en_width, module->Mux(NEW_ID, orig_bram_dout, bypass_dout, transp_en_q[i])); - } + if (pi.mapped_port == -1) + { + if (pi.enable) + c->setPort(stringf("\\%sEN", pf), State::S0); + continue; } + auto &port = mem.rd_ports[pi.mapped_port]; + SigSpec sig_data = port.data.extract(grid_d * bram.dbits, bram.dbits); - for (int i = bram.dbits-1; i >= 0; i--) - if (sig_data[i].wire == nullptr) { - sig_data.remove(i); - bram_dout.remove(i); - } + SigSpec bram_dout = module->addWire(NEW_ID, bram.dbits); + c->setPort(stringf("\\%sDATA", pf), bram_dout); SigSpec addr_ok_q = addr_ok; - if ((pi.clocks || pi.make_outreg) && !addr_ok.empty()) { + if (port.clk_enable && !addr_ok.empty()) { addr_ok_q = module->addWire(NEW_ID); - if (!pi.sig_en.empty()) - addr_ok = module->Mux(NEW_ID, addr_ok_q, addr_ok, pi.sig_en); - module->addDff(NEW_ID, pi.sig_clock, addr_ok, addr_ok_q, pi.effective_clkpol); + module->addDffe(NEW_ID, port.clk, port.en, addr_ok, addr_ok_q, port.clk_polarity); } dout_cache[sig_data].first.append(addr_ok_q); dout_cache[sig_data].second.append(bram_dout); + + if (pi.enable) { + SigSpec sig_en = port.en; + if (!addr_ok.empty()) + sig_en = module->And(NEW_ID, sig_en, addr_ok); + c->setPort(stringf("\\%sEN", pf), sig_en); + } } } } @@ -1063,22 +998,23 @@ grow_read_ports:; } } - module->remove(cell); + mem.remove(); return true; } -void handle_cell(Cell *cell, const rules_t &rules) +void handle_memory(Mem &mem, const rules_t &rules, FfInitVals *initvals) { - log("Processing %s.%s:\n", log_id(cell->module), log_id(cell)); + log("Processing %s.%s:\n", log_id(mem.module), log_id(mem.memid)); + mem.narrow(); - bool cell_init = !SigSpec(cell->getParam(ID::INIT)).is_fully_undef(); + bool cell_init = !mem.inits.empty(); dict<string, int> match_properties; - match_properties["words"] = cell->getParam(ID::SIZE).as_int(); - match_properties["abits"] = cell->getParam(ID::ABITS).as_int(); - match_properties["dbits"] = cell->getParam(ID::WIDTH).as_int(); - match_properties["wports"] = cell->getParam(ID::WR_PORTS).as_int(); - match_properties["rports"] = cell->getParam(ID::RD_PORTS).as_int(); + match_properties["words"] = mem.size; + match_properties["abits"] = ceil_log2(mem.size); + match_properties["dbits"] = mem.width; + match_properties["wports"] = GetSize(mem.wr_ports); + match_properties["rports"] = GetSize(mem.rd_ports); match_properties["bits"] = match_properties["words"] * match_properties["dbits"]; match_properties["ports"] = match_properties["wports"] + match_properties["rports"]; @@ -1181,8 +1117,8 @@ void handle_cell(Cell *cell, const rules_t &rules) bool exists = std::get<0>(term); IdString key = std::get<1>(term); const Const &value = std::get<2>(term); - auto it = cell->attributes.find(key); - if (it == cell->attributes.end()) { + auto it = mem.attributes.find(key); + if (it == mem.attributes.end()) { if (exists) continue; found = true; @@ -1219,7 +1155,7 @@ void handle_cell(Cell *cell, const rules_t &rules) if (or_next_if_better && i+1 == GetSize(rules.matches) && vi+1 == GetSize(rules.brams.at(match.name))) log_error("Found 'or_next_if_better' in last match rule.\n"); - if (!replace_cell(cell, rules, bram, match, match_properties, 1)) { + if (!replace_memory(mem, rules, initvals, bram, match, match_properties, 1)) { log(" Mapping to bram type %s failed.\n", log_id(match.name)); failed_brams.insert(pair<IdString, int>(bram.name, bram.variant)); goto next_match_rule; @@ -1246,12 +1182,12 @@ void handle_cell(Cell *cell, const rules_t &rules) best_rule_cache.clear(); auto &best_bram = rules.brams.at(rules.matches.at(best_rule.first).name).at(best_rule.second); - if (!replace_cell(cell, rules, best_bram, rules.matches.at(best_rule.first), match_properties, 2)) + if (!replace_memory(mem, rules, initvals, best_bram, rules.matches.at(best_rule.first), match_properties, 2)) log_error("Mapping to bram type %s (variant %d) after pre-selection failed.\n", log_id(best_bram.name), best_bram.variant); return; } - if (!replace_cell(cell, rules, bram, match, match_properties, 0)) { + if (!replace_memory(mem, rules, initvals, bram, match, match_properties, 0)) { log(" Mapping to bram type %s failed.\n", log_id(match.name)); failed_brams.insert(pair<IdString, int>(bram.name, bram.variant)); goto next_match_rule; @@ -1383,10 +1319,12 @@ struct MemoryBramPass : public Pass { } extra_args(args, argidx, design); - for (auto mod : design->selected_modules()) - for (auto cell : mod->selected_cells()) - if (cell->type == ID($mem)) - handle_cell(cell, rules); + for (auto mod : design->selected_modules()) { + SigMap sigmap(mod); + FfInitVals initvals(&sigmap, mod); + for (auto &mem : Mem::get_selected_memories(mod)) + handle_memory(mem, rules, &initvals); + } } } MemoryBramPass; diff --git a/passes/memory/memory_collect.cc b/passes/memory/memory_collect.cc index 7e82f47dc..bf3bb34f8 100644 --- a/passes/memory/memory_collect.cc +++ b/passes/memory/memory_collect.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,231 +18,11 @@ */ #include "kernel/yosys.h" -#include "kernel/sigtools.h" +#include "kernel/mem.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN -bool memcells_cmp(Cell *a, Cell *b) -{ - if (a->type == ID($memrd) && b->type == ID($memrd)) - return a->name < b->name; - if (a->type == ID($memrd) || b->type == ID($memrd)) - return (a->type == ID($memrd)) < (b->type == ID($memrd)); - return a->parameters.at(ID::PRIORITY).as_int() < b->parameters.at(ID::PRIORITY).as_int(); -} - -Cell *handle_memory(Module *module, RTLIL::Memory *memory) -{ - log("Collecting $memrd, $memwr and $meminit for memory `%s' in module `%s':\n", - memory->name.c_str(), module->name.c_str()); - - Const init_data(State::Sx, memory->size * memory->width); - SigMap sigmap(module); - - int wr_ports = 0; - SigSpec sig_wr_clk; - SigSpec sig_wr_clk_enable; - SigSpec sig_wr_clk_polarity; - SigSpec sig_wr_addr; - SigSpec sig_wr_data; - SigSpec sig_wr_en; - - int rd_ports = 0; - SigSpec sig_rd_clk; - SigSpec sig_rd_clk_enable; - SigSpec sig_rd_clk_polarity; - SigSpec sig_rd_transparent; - SigSpec sig_rd_addr; - SigSpec sig_rd_data; - SigSpec sig_rd_en; - - int addr_bits = 0; - std::vector<Cell*> memcells; - - for (auto cell : module->cells()) - if (cell->type.in(ID($memrd), ID($memwr), ID($meminit)) && memory->name == cell->parameters[ID::MEMID].decode_string()) { - SigSpec addr = sigmap(cell->getPort(ID::ADDR)); - for (int i = 0; i < GetSize(addr); i++) - if (addr[i] != State::S0) - addr_bits = std::max(addr_bits, i+1); - memcells.push_back(cell); - } - - if (memory->start_offset == 0 && addr_bits < 30 && (1 << addr_bits) < memory->size) - memory->size = 1 << addr_bits; - - if (memory->start_offset >= 0) - addr_bits = std::min(addr_bits, ceil_log2(memory->size + memory->start_offset)); - - addr_bits = std::max(addr_bits, 1); - - if (memcells.empty()) { - log(" no cells found. removing memory.\n"); - return nullptr; - } - - std::sort(memcells.begin(), memcells.end(), memcells_cmp); - - for (auto cell : memcells) - { - log(" %s (%s)\n", log_id(cell), log_id(cell->type)); - - if (cell->type == ID($meminit)) - { - SigSpec addr = sigmap(cell->getPort(ID::ADDR)); - SigSpec data = sigmap(cell->getPort(ID::DATA)); - - if (!addr.is_fully_const()) - log_error("Non-constant address %s in memory initialization %s.\n", log_signal(addr), log_id(cell)); - if (!data.is_fully_const()) - log_error("Non-constant data %s in memory initialization %s.\n", log_signal(data), log_id(cell)); - - int offset = (addr.as_int() - memory->start_offset) * memory->width; - - if (offset < 0 || offset + GetSize(data) > GetSize(init_data)) - log_warning("Address %s in memory initialization %s is out-of-bounds.\n", log_signal(addr), log_id(cell)); - - for (int i = 0; i < GetSize(data); i++) - if (0 <= i+offset && i+offset < GetSize(init_data)) - init_data.bits[i+offset] = data[i].data; - - continue; - } - - if (cell->type == ID($memwr)) - { - SigSpec clk = sigmap(cell->getPort(ID::CLK)); - SigSpec clk_enable = SigSpec(cell->parameters[ID::CLK_ENABLE]); - SigSpec clk_polarity = SigSpec(cell->parameters[ID::CLK_POLARITY]); - SigSpec addr = sigmap(cell->getPort(ID::ADDR)); - SigSpec data = sigmap(cell->getPort(ID::DATA)); - SigSpec en = sigmap(cell->getPort(ID::EN)); - - if (!en.is_fully_zero()) - { - clk.extend_u0(1, false); - clk_enable.extend_u0(1, false); - clk_polarity.extend_u0(1, false); - addr.extend_u0(addr_bits, false); - data.extend_u0(memory->width, false); - en.extend_u0(memory->width, false); - - sig_wr_clk.append(clk); - sig_wr_clk_enable.append(clk_enable); - sig_wr_clk_polarity.append(clk_polarity); - sig_wr_addr.append(addr); - sig_wr_data.append(data); - sig_wr_en.append(en); - - wr_ports++; - } - continue; - } - - if (cell->type == ID($memrd)) - { - SigSpec clk = sigmap(cell->getPort(ID::CLK)); - SigSpec clk_enable = SigSpec(cell->parameters[ID::CLK_ENABLE]); - SigSpec clk_polarity = SigSpec(cell->parameters[ID::CLK_POLARITY]); - SigSpec transparent = SigSpec(cell->parameters[ID::TRANSPARENT]); - SigSpec addr = sigmap(cell->getPort(ID::ADDR)); - SigSpec data = sigmap(cell->getPort(ID::DATA)); - SigSpec en = sigmap(cell->getPort(ID::EN)); - - if (!en.is_fully_zero()) - { - clk.extend_u0(1, false); - clk_enable.extend_u0(1, false); - clk_polarity.extend_u0(1, false); - transparent.extend_u0(1, false); - addr.extend_u0(addr_bits, false); - data.extend_u0(memory->width, false); - - sig_rd_clk.append(clk); - sig_rd_clk_enable.append(clk_enable); - sig_rd_clk_polarity.append(clk_polarity); - sig_rd_transparent.append(transparent); - sig_rd_addr.append(addr); - sig_rd_data.append(data); - sig_rd_en.append(en); - - rd_ports++; - } - continue; - } - } - - std::stringstream sstr; - sstr << "$mem$" << memory->name.str() << "$" << (autoidx++); - - Cell *mem = module->addCell(sstr.str(), ID($mem)); - mem->parameters[ID::MEMID] = Const(memory->name.str()); - mem->parameters[ID::WIDTH] = Const(memory->width); - mem->parameters[ID::OFFSET] = Const(memory->start_offset); - mem->parameters[ID::SIZE] = Const(memory->size); - mem->parameters[ID::ABITS] = Const(addr_bits); - mem->parameters[ID::INIT] = init_data; - - log_assert(sig_wr_clk.size() == wr_ports); - log_assert(sig_wr_clk_enable.size() == wr_ports && sig_wr_clk_enable.is_fully_const()); - log_assert(sig_wr_clk_polarity.size() == wr_ports && sig_wr_clk_polarity.is_fully_const()); - log_assert(sig_wr_addr.size() == wr_ports * addr_bits); - log_assert(sig_wr_data.size() == wr_ports * memory->width); - log_assert(sig_wr_en.size() == wr_ports * memory->width); - - mem->parameters[ID::WR_PORTS] = Const(wr_ports); - mem->parameters[ID::WR_CLK_ENABLE] = wr_ports ? sig_wr_clk_enable.as_const() : State::S0; - mem->parameters[ID::WR_CLK_POLARITY] = wr_ports ? sig_wr_clk_polarity.as_const() : State::S0; - - mem->setPort(ID::WR_CLK, sig_wr_clk); - mem->setPort(ID::WR_ADDR, sig_wr_addr); - mem->setPort(ID::WR_DATA, sig_wr_data); - mem->setPort(ID::WR_EN, sig_wr_en); - - log_assert(sig_rd_clk.size() == rd_ports); - log_assert(sig_rd_clk_enable.size() == rd_ports && sig_rd_clk_enable.is_fully_const()); - log_assert(sig_rd_clk_polarity.size() == rd_ports && sig_rd_clk_polarity.is_fully_const()); - log_assert(sig_rd_addr.size() == rd_ports * addr_bits); - log_assert(sig_rd_data.size() == rd_ports * memory->width); - - mem->parameters[ID::RD_PORTS] = Const(rd_ports); - mem->parameters[ID::RD_CLK_ENABLE] = rd_ports ? sig_rd_clk_enable.as_const() : State::S0; - mem->parameters[ID::RD_CLK_POLARITY] = rd_ports ? sig_rd_clk_polarity.as_const() : State::S0; - mem->parameters[ID::RD_TRANSPARENT] = rd_ports ? sig_rd_transparent.as_const() : State::S0; - - mem->setPort(ID::RD_CLK, sig_rd_clk); - mem->setPort(ID::RD_ADDR, sig_rd_addr); - mem->setPort(ID::RD_DATA, sig_rd_data); - mem->setPort(ID::RD_EN, sig_rd_en); - - // Copy attributes from RTLIL memory to $mem - for (auto attr : memory->attributes) - mem->attributes[attr.first] = attr.second; - - for (auto c : memcells) - module->remove(c); - - return mem; -} - -static void handle_module(Design *design, Module *module) -{ - std::vector<pair<Cell*, IdString>> finqueue; - - for (auto &mem_it : module->memories) - if (design->selected(module, mem_it.second)) { - Cell *c = handle_memory(module, mem_it.second); - finqueue.push_back(pair<Cell*, IdString>(c, mem_it.first)); - } - for (auto &it : finqueue) { - delete module->memories.at(it.second); - module->memories.erase(it.second); - if (it.first) - module->rename(it.first, it.second); - } -} - struct MemoryCollectPass : public Pass { MemoryCollectPass() : Pass("memory_collect", "creating multi-port memory cells") { } void help() override @@ -258,8 +38,14 @@ struct MemoryCollectPass : public Pass { void execute(std::vector<std::string> args, RTLIL::Design *design) override { log_header(design, "Executing MEMORY_COLLECT pass (generating $mem cells).\n"); extra_args(args, 1, design); - for (auto module : design->selected_modules()) - handle_module(design, module); + for (auto module : design->selected_modules()) { + for (auto &mem : Mem::get_selected_memories(module)) { + if (!mem.packed) { + mem.packed = true; + mem.emit(); + } + } + } } } MemoryCollectPass; diff --git a/passes/memory/memory_dff.cc b/passes/memory/memory_dff.cc index 947cf7b35..91209d428 100644 --- a/passes/memory/memory_dff.cc +++ b/passes/memory/memory_dff.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,313 +17,627 @@ * */ -#include <algorithm> #include "kernel/yosys.h" #include "kernel/sigtools.h" +#include "kernel/modtools.h" +#include "kernel/ffinit.h" +#include "kernel/qcsat.h" +#include "kernel/mem.h" +#include "kernel/ff.h" +#include "kernel/ffmerge.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN -struct MemoryDffWorker -{ - Module *module; - SigMap sigmap; +struct MuxData { + int base_idx; + int size; + bool is_b; + SigSpec sig_s; + std::vector<SigSpec> sig_other; +}; - vector<Cell*> dff_cells; - dict<SigBit, SigBit> invbits; - dict<SigBit, int> sigbit_users_count; - dict<SigSpec, Cell*> mux_cells_a, mux_cells_b; - pool<Cell*> forward_merged_dffs, candidate_dffs; - pool<SigBit> init_bits; +struct PortData { + bool relevant; + std::vector<bool> uncollidable_mask; + std::vector<bool> transparency_mask; + std::vector<bool> collision_x_mask; + bool final_transparency; + bool final_collision_x; +}; - MemoryDffWorker(Module *module) : module(module), sigmap(module) - { - for (auto wire : module->wires()) { - if (wire->attributes.count(ID::init) == 0) - continue; - SigSpec sig = sigmap(wire); - Const initval = wire->attributes.at(ID::init); - for (int i = 0; i < GetSize(sig) && i < GetSize(initval); i++) - if (initval[i] == State::S0 || initval[i] == State::S1) - init_bits.insert(sig[i]); +// A helper with some caching for transparency-related SAT queries. +// Bound to a single memory read port in the process of being converted +// from async to sync.. +struct MemQueryCache +{ + QuickConeSat &qcsat; + // The memory. + Mem &mem; + // The port, still async at this point. + MemRd &port; + // The virtual FF that will end up merged into this port. + FfData &ff; + // An ezSAT variable that is true when we actually care about the data + // read from memory (ie. the FF has enable on and is not in reset). + int port_ren; + // Some caches. + dict<std::pair<int, SigBit>, bool> cache_can_collide_rdwr; + dict<std::tuple<int, int, SigBit, SigBit>, bool> cache_can_collide_together; + dict<std::tuple<int, SigBit, SigBit, bool>, bool> cache_is_w2rbyp; + dict<std::tuple<SigBit, bool>, bool> cache_impossible_with_ren; + + MemQueryCache(QuickConeSat &qcsat, Mem &mem, MemRd &port, FfData &ff) : qcsat(qcsat), mem(mem), port(port), ff(ff) { + // port_ren is an upper bound on when we care about the value fetched + // from memory this cycle. + int ren = ezSAT::CONST_TRUE; + if (ff.has_ce) { + ren = qcsat.importSigBit(ff.sig_ce); + if (!ff.pol_ce) + ren = qcsat.ez->NOT(ren); + } + if (ff.has_srst) { + int nrst = qcsat.importSigBit(ff.sig_srst); + if (ff.pol_srst) + nrst = qcsat.ez->NOT(nrst); + ren = qcsat.ez->AND(ren, nrst); } + port_ren = ren; } - bool find_sig_before_dff(RTLIL::SigSpec &sig, RTLIL::SigSpec &clk, bool &clk_polarity, bool after = false) - { - sigmap.apply(sig); - - for (auto &bit : sig) - { - if (bit.wire == NULL) - continue; - - if (!after && init_bits.count(sigmap(bit))) - return false; - - for (auto cell : dff_cells) - { - if (after && forward_merged_dffs.count(cell)) - continue; - - SigSpec this_clk = cell->getPort(ID::CLK); - bool this_clk_polarity = cell->parameters[ID::CLK_POLARITY].as_bool(); - - if (invbits.count(this_clk)) { - this_clk = invbits.at(this_clk); - this_clk_polarity = !this_clk_polarity; - } - - if (clk != RTLIL::SigSpec(RTLIL::State::Sx)) { - if (this_clk != clk) - continue; - if (this_clk_polarity != clk_polarity) - continue; - } + // Returns ezSAT variable that is true iff the two addresses are the same. + int addr_eq(SigSpec raddr, SigSpec waddr) { + int abits = std::max(GetSize(raddr), GetSize(waddr)); + raddr.extend_u0(abits); + waddr.extend_u0(abits); + return qcsat.ez->vec_eq(qcsat.importSig(raddr), qcsat.importSig(waddr)); + } - RTLIL::SigSpec q_norm = cell->getPort(after ? ID::D : ID::Q); - sigmap.apply(q_norm); + // Returns true if a given write port bit can be active at the same time + // as this read port and at the same address. + bool can_collide_rdwr(int widx, SigBit wen) { + std::pair<int, SigBit> key(widx, wen); + auto it = cache_can_collide_rdwr.find(key); + if (it != cache_can_collide_rdwr.end()) + return it->second; + auto &wport = mem.wr_ports[widx]; + int aeq = addr_eq(port.addr, wport.addr); + int wen_sat = qcsat.importSigBit(wen); + qcsat.prepare(); + bool res = qcsat.ez->solve(aeq, wen_sat, port_ren); + cache_can_collide_rdwr[key] = res; + return res; + } - RTLIL::SigSpec d = q_norm.extract(bit, &cell->getPort(after ? ID::Q : ID::D)); - if (d.size() != 1) - continue; + // Returns true if both given write port bits can be active at the same + // time as this read port and at the same address (three-way collision). + bool can_collide_together(int widx1, int widx2, int bitidx) { + auto &wport1 = mem.wr_ports[widx1]; + auto &wport2 = mem.wr_ports[widx2]; + SigBit wen1 = wport1.en[bitidx]; + SigBit wen2 = wport2.en[bitidx]; + std::tuple<int, int, SigBit, SigBit> key(widx1, widx2, wen1, wen2); + auto it = cache_can_collide_together.find(key); + if (it != cache_can_collide_together.end()) + return it->second; + int aeq1 = addr_eq(port.addr, wport1.addr); + int aeq2 = addr_eq(port.addr, wport2.addr); + int wen1_sat = qcsat.importSigBit(wen1); + int wen2_sat = qcsat.importSigBit(wen2); + qcsat.prepare(); + bool res = qcsat.ez->solve(wen1_sat, wen2_sat, aeq1, aeq2, port_ren); + cache_can_collide_together[key] = res; + return res; + } - if (after && init_bits.count(d)) - return false; + // Returns true if the given mux selection signal is a valid data-bypass + // signal in soft transparency logic for a given write port bit. + bool is_w2rbyp(int widx, SigBit wen, SigBit sel, bool neg_sel) { + std::tuple<int, SigBit, SigBit, bool> key(widx, wen, sel, neg_sel); + auto it = cache_is_w2rbyp.find(key); + if (it != cache_is_w2rbyp.end()) + return it->second; + auto &wport = mem.wr_ports[widx]; + int aeq = addr_eq(port.addr, wport.addr); + int wen_sat = qcsat.importSigBit(wen); + int sel_expected = qcsat.ez->AND(aeq, wen_sat); + int sel_sat = qcsat.importSigBit(sel); + if (neg_sel) + sel_sat = qcsat.ez->NOT(sel_sat); + qcsat.prepare(); + bool res = !qcsat.ez->solve(port_ren, qcsat.ez->XOR(sel_expected, sel_sat)); + cache_is_w2rbyp[key] = res; + return res; + } - bit = d; - clk = this_clk; - clk_polarity = this_clk_polarity; - candidate_dffs.insert(cell); - goto replaced_this_bit; - } + // Returns true if the given mux selection signal can never be true + // when this port is active. + bool impossible_with_ren(SigBit sel, bool neg_sel) { + std::tuple<SigBit, bool> key(sel, neg_sel); + auto it = cache_impossible_with_ren.find(key); + if (it != cache_impossible_with_ren.end()) + return it->second; + int sel_sat = qcsat.importSigBit(sel); + if (neg_sel) + sel_sat = qcsat.ez->NOT(sel_sat); + qcsat.prepare(); + bool res = !qcsat.ez->solve(port_ren, sel_sat); + cache_impossible_with_ren[key] = res; + return res; + } + // Helper for data_eq: walks up a multiplexer when the value of its + // sel signal is constant under the assumption that this read port + // is active and a given other mux sel signal is true. + bool walk_up_mux_cond(SigBit sel, bool neg_sel, SigBit &bit) { + auto &drivers = qcsat.modwalker.signal_drivers[qcsat.modwalker.sigmap(bit)]; + if (GetSize(drivers) != 1) return false; - replaced_this_bit:; + auto driver = *drivers.begin(); + if (!driver.cell->type.in(ID($mux), ID($pmux))) + return false; + log_assert(driver.port == ID::Y); + SigSpec sig_s = driver.cell->getPort(ID::S); + int sel_sat = qcsat.importSigBit(sel); + if (neg_sel) + sel_sat = qcsat.ez->NOT(sel_sat); + bool all_0 = true; + int width = driver.cell->parameters.at(ID::WIDTH).as_int(); + for (int i = 0; i < GetSize(sig_s); i++) { + int sbit = qcsat.importSigBit(sig_s[i]); + qcsat.prepare(); + if (!qcsat.ez->solve(port_ren, sel_sat, qcsat.ez->NOT(sbit))) { + bit = driver.cell->getPort(ID::B)[i * width + driver.offset]; + return true; + } + if (qcsat.ez->solve(port_ren, sel_sat, sbit)) + all_0 = false; } + if (all_0) { + bit = driver.cell->getPort(ID::A)[driver.offset]; + return true; + } + return false; + } - return true; + // Returns true if a given data signal is equivalent to another, under + // the assumption that this read port is active and a given mux sel signal + // is true. Used to match transparency logic data with write port data. + // The walk_up_mux_cond part is necessary because write ports in yosys + // tend to be connected to things like (wen ? wdata : 'x). + bool data_eq(SigBit sel, bool neg_sel, SigBit dbit, SigBit odbit) { + if (qcsat.modwalker.sigmap(dbit) == qcsat.modwalker.sigmap(odbit)) + return true; + while (walk_up_mux_cond(sel, neg_sel, dbit)); + while (walk_up_mux_cond(sel, neg_sel, odbit)); + return qcsat.modwalker.sigmap(dbit) == qcsat.modwalker.sigmap(odbit); } +}; - void handle_wr_cell(RTLIL::Cell *cell) +struct MemoryDffWorker +{ + Module *module; + ModWalker modwalker; + FfInitVals initvals; + FfMergeHelper merger; + + MemoryDffWorker(Module *module) : module(module), modwalker(module->design) { - log("Checking cell `%s' in module `%s': ", cell->name.c_str(), module->name.c_str()); + modwalker.setup(module); + initvals.set(&modwalker.sigmap, module); + merger.set(&initvals, module); + } - RTLIL::SigSpec clk = RTLIL::SigSpec(RTLIL::State::Sx); - bool clk_polarity = 0; - candidate_dffs.clear(); + // Starting from the output of an async read port, as long as the data + // signal's only user is a mux data signal, passes through the mux + // and remembers information about it. Conceptually works on every + // bit separately, but coalesces the result when possible. + SigSpec walk_muxes(SigSpec data, std::vector<MuxData> &res) { + bool did_something; + do { + did_something = false; + int prev_idx = -1; + Cell *prev_cell = nullptr; + bool prev_is_b = false; + for (int i = 0; i < GetSize(data); i++) { + SigBit bit = modwalker.sigmap(data[i]); + auto &consumers = modwalker.signal_consumers[bit]; + if (GetSize(consumers) != 1 || modwalker.signal_outputs.count(bit)) + continue; + auto consumer = *consumers.begin(); + bool is_b; + if (consumer.cell->type == ID($mux)) { + if (consumer.port == ID::A) { + is_b = false; + } else if (consumer.port == ID::B) { + is_b = true; + } else { + continue; + } + } else if (consumer.cell->type == ID($pmux)) { + if (consumer.port == ID::A) { + is_b = false; + } else { + continue; + } + } else { + continue; + } + SigSpec y = consumer.cell->getPort(ID::Y); + int mux_width = GetSize(y); + SigBit ybit = y.extract(consumer.offset); + if (prev_cell != consumer.cell || prev_idx+1 != i || prev_is_b != is_b) { + MuxData md; + md.base_idx = i; + md.size = 0; + md.is_b = is_b; + md.sig_s = consumer.cell->getPort(ID::S); + md.sig_other.resize(GetSize(md.sig_s)); + prev_cell = consumer.cell; + prev_is_b = is_b; + res.push_back(md); + } + auto &md = res.back(); + md.size++; + for (int j = 0; j < GetSize(md.sig_s); j++) { + SigBit obit = consumer.cell->getPort(is_b ? ID::A : ID::B).extract(j * mux_width + consumer.offset); + md.sig_other[j].append(obit); + } + prev_idx = i; + data[i] = ybit; + did_something = true; + } + } while (did_something); + return data; + } - RTLIL::SigSpec sig_addr = cell->getPort(ID::ADDR); - if (!find_sig_before_dff(sig_addr, clk, clk_polarity)) { - log("no (compatible) $dff for address input found.\n"); + // Merges FF and possibly soft transparency logic into an asynchronous + // read port, making it into a synchronous one. + // + // There are three moving parts involved here: + // + // - the async port, which we start from, whose data port is input to... + // - an arbitrary chain of $mux and $pmux cells implementing soft transparency + // logic (ie. bypassing write port's data iff the write port is active and + // writing to the same address as this read port), which in turn feeds... + // - a final FF + // + // The async port and the mux chain are not allowed to have any users that + // are not part of the above. + // + // The algorithm is: + // + // 1. Walk through the muxes. + // 2. Recognize the final FF. + // 3. Knowing the FF's clock and read enable, make a list of write ports + // that we'll run transparency analysis on. + // 4. For every mux bit, recognize it as one of: + // - a transparency bypass mux for some port + // - a bypass mux that feeds 'x instead (this will result in collision + // don't care behavior being recognized) + // - a mux that never selects the other value when read port is active, + // and can thus be skipped (this is necessary because this could + // be a transparency bypass mux for never-colliding port that other + // passes failed to optimize) + // - a mux whose other input is 'x, and can thus be skipped + // 5. When recognizing transparency bypasses, take care to preserve priority + // behavior — when two bypasses are sequential muxes on the chain, they + // effectively have priority over one another, and the transform can + // only be performed when either a) their corresponding write ports + // also have priority, or b) there can never be a three-way collision + // between the two write ports and the read port. + // 6. Check consistency of per-bit transparency masks, merge them into + // per-port transparency masks + // 7. If everything went fine in the previous steps, actually perform + // the merge. + void handle_rd_port(Mem &mem, QuickConeSat &qcsat, int idx) + { + auto &port = mem.rd_ports[idx]; + log("Checking read port `%s'[%d] in module `%s': ", mem.memid.c_str(), idx, module->name.c_str()); + + std::vector<MuxData> muxdata; + SigSpec data = walk_muxes(port.data, muxdata); + FfData ff; + pool<std::pair<Cell *, int>> bits; + if (!merger.find_output_ff(data, ff, bits)) { + log("no output FF found.\n"); return; } - - RTLIL::SigSpec sig_data = cell->getPort(ID::DATA); - if (!find_sig_before_dff(sig_data, clk, clk_polarity)) { - log("no (compatible) $dff for data input found.\n"); + if (!ff.has_clk) { + log("output latches are not supported.\n"); return; } - - RTLIL::SigSpec sig_en = cell->getPort(ID::EN); - if (!find_sig_before_dff(sig_en, clk, clk_polarity)) { - log("no (compatible) $dff for enable input found.\n"); + if (ff.has_aload) { + log("output FF has async load, not supported.\n"); return; } - - if (clk != RTLIL::SigSpec(RTLIL::State::Sx)) - { - for (auto cell : candidate_dffs) - forward_merged_dffs.insert(cell); - - cell->setPort(ID::CLK, clk); - cell->setPort(ID::ADDR, sig_addr); - cell->setPort(ID::DATA, sig_data); - cell->setPort(ID::EN, sig_en); - cell->parameters[ID::CLK_ENABLE] = RTLIL::Const(1); - cell->parameters[ID::CLK_POLARITY] = RTLIL::Const(clk_polarity); - - log("merged $dff to cell.\n"); + if (ff.has_sr) { + // Latches and FFs with SR are not supported. + log("output FF has both set and reset, not supported.\n"); return; } - log("no (compatible) $dff found.\n"); - } - - void disconnect_dff(RTLIL::SigSpec sig) - { - sigmap.apply(sig); - sig.sort_and_unify(); - - std::stringstream sstr; - sstr << "$memory_dff_disconnected$" << (autoidx++); - - RTLIL::SigSpec new_sig = module->addWire(sstr.str(), sig.size()); - - for (auto cell : module->cells()) - if (cell->type == ID($dff)) { - RTLIL::SigSpec new_q = cell->getPort(ID::Q); - new_q.replace(sig, new_sig); - cell->setPort(ID::Q, new_q); + // Construct cache. + MemQueryCache cache(qcsat, mem, port, ff); + + // Prepare information structure about all ports, recognize port bits + // that can never collide at all and don't need to be checked. + std::vector<PortData> portdata; + for (int i = 0; i < GetSize(mem.wr_ports); i++) { + PortData pd; + auto &wport = mem.wr_ports[i]; + pd.relevant = true; + if (!wport.clk_enable) + pd.relevant = false; + if (wport.clk != ff.sig_clk) + pd.relevant = false; + if (wport.clk_polarity != ff.pol_clk) + pd.relevant = false; + // In theory, we *could* support mismatched width + // ports here. However, it's not worth it — wide + // ports are recognized *after* memory_dff in + // a normal flow. + if (wport.wide_log2 != port.wide_log2) + pd.relevant = false; + pd.uncollidable_mask.resize(GetSize(port.data)); + pd.transparency_mask.resize(GetSize(port.data)); + pd.collision_x_mask.resize(GetSize(port.data)); + if (pd.relevant) { + // If we got this far, this port is potentially + // transparent and/or has undefined collision + // behavior. Now, for every bit, check if it can + // ever collide. + for (int j = 0; j < ff.width; j++) { + if (!cache.can_collide_rdwr(i, wport.en[j])) { + pd.uncollidable_mask[j] = true; + pd.collision_x_mask[j] = true; + } + } } - } - - void handle_rd_cell(RTLIL::Cell *cell) - { - log("Checking cell `%s' in module `%s': ", cell->name.c_str(), module->name.c_str()); - - bool clk_polarity = 0; - - RTLIL::SigSpec clk_data = RTLIL::SigSpec(RTLIL::State::Sx); - RTLIL::SigSpec sig_data = cell->getPort(ID::DATA); - - for (auto bit : sigmap(sig_data)) - if (sigbit_users_count[bit] > 1) - goto skip_ff_after_read_merging; - - if (mux_cells_a.count(sig_data) || mux_cells_b.count(sig_data)) - { - RTLIL::SigSpec en; - std::vector<RTLIL::SigSpec> check_q; - - do { - bool enable_invert = mux_cells_a.count(sig_data) != 0; - Cell *mux = enable_invert ? mux_cells_a.at(sig_data) : mux_cells_b.at(sig_data); - check_q.push_back(sigmap(mux->getPort(enable_invert ? ID::B : ID::A))); - sig_data = sigmap(mux->getPort(ID::Y)); - en.append(enable_invert ? module->LogicNot(NEW_ID, mux->getPort(ID::S)) : mux->getPort(ID::S)); - } while (mux_cells_a.count(sig_data) || mux_cells_b.count(sig_data)); + portdata.push_back(pd); + } - for (auto bit : sig_data) - if (sigbit_users_count[bit] > 1) - goto skip_ff_after_read_merging; + // Now inspect the mux chain. + for (auto &md : muxdata) { + // We only mark transparent bits after processing a complete + // mux, so that the transparency priority validation check + // below sees transparency information as of previous mux. + std::vector<std::pair<PortData&, int>> trans_queue; + for (int sel_idx = 0; sel_idx < GetSize(md.sig_s); sel_idx++) { + SigBit sbit = md.sig_s[sel_idx]; + SigSpec &odata = md.sig_other[sel_idx]; + for (int bitidx = md.base_idx; bitidx < md.base_idx+md.size; bitidx++) { + SigBit odbit = odata[bitidx-md.base_idx]; + bool recognized = false; + for (int pi = 0; pi < GetSize(mem.wr_ports); pi++) { + auto &pd = portdata[pi]; + auto &wport = mem.wr_ports[pi]; + if (!pd.relevant) + continue; + if (pd.uncollidable_mask[bitidx]) + continue; + bool match = cache.is_w2rbyp(pi, wport.en[bitidx], sbit, md.is_b); + if (!match) + continue; + // If we got here, we recognized this mux sel + // as valid bypass sel for a given port bit. + if (odbit == State::Sx) { + // 'x, mark collision don't care. + pd.collision_x_mask[bitidx] = true; + pd.transparency_mask[bitidx] = false; + } else if (cache.data_eq(sbit, md.is_b, wport.data[bitidx], odbit)) { + // Correct data value, mark transparency, + // but only after verifying that priority + // is fine. + for (int k = 0; k < GetSize(mem.wr_ports); k++) { + if (portdata[k].transparency_mask[bitidx]) { + if (wport.priority_mask[k]) + continue; + if (!cache.can_collide_together(pi, k, bitidx)) + continue; + log("FF found, but transparency logic priority doesn't match write priority.\n"); + return; + } + } + recognized = true; + trans_queue.push_back({pd, bitidx}); + break; + } else { + log("FF found, but with a mux data input that doesn't seem to correspond to transparency logic.\n"); + return; + } + } + if (!recognized) { + // If we haven't positively identified this as + // a bypass: it's still skippable if the + // data is 'x, or if the sel cannot actually be + // active. + if (odbit == State::Sx) + continue; + if (cache.impossible_with_ren(sbit, md.is_b)) + continue; + log("FF found, but with a mux select that doesn't seem to correspond to transparency logic.\n"); + return; + } + } + } + // Done with this mux, now actually apply the transparencies. + for (auto it : trans_queue) { + it.first.transparency_mask[it.second] = true; + it.first.collision_x_mask[it.second] = false; + } + } - if (find_sig_before_dff(sig_data, clk_data, clk_polarity, true) && clk_data != RTLIL::SigSpec(RTLIL::State::Sx) && - std::all_of(check_q.begin(), check_q.end(), [&](const SigSpec &cq) {return cq == sig_data; })) - { - disconnect_dff(sig_data); - cell->setPort(ID::CLK, clk_data); - cell->setPort(ID::EN, en.size() > 1 ? module->ReduceAnd(NEW_ID, en) : en); - cell->setPort(ID::DATA, sig_data); - cell->parameters[ID::CLK_ENABLE] = RTLIL::Const(1); - cell->parameters[ID::CLK_POLARITY] = RTLIL::Const(clk_polarity); - cell->parameters[ID::TRANSPARENT] = RTLIL::Const(0); - log("merged data $dff with rd enable to cell.\n"); + // Final merging and validation of per-bit masks. + for (int pi = 0; pi < GetSize(mem.wr_ports); pi++) { + auto &pd = portdata[pi]; + if (!pd.relevant) + continue; + bool trans = false; + bool non_trans = false; + for (int i = 0; i < ff.width; i++) { + if (pd.collision_x_mask[i]) + continue; + if (pd.transparency_mask[i]) + trans = true; + else + non_trans = true; + } + if (trans && non_trans) { + log("FF found, but soft transparency logic is inconsistent for port %d.\n", pi); return; } + pd.final_transparency = trans; + pd.final_collision_x = !trans && !non_trans; } + + // OK, it worked. + log("merging output FF to cell.\n"); + + merger.remove_output_ff(bits); + if (ff.has_ce && !ff.pol_ce) + ff.sig_ce = module->LogicNot(NEW_ID, ff.sig_ce); + if (ff.has_arst && !ff.pol_arst) + ff.sig_arst = module->LogicNot(NEW_ID, ff.sig_arst); + if (ff.has_srst && !ff.pol_srst) + ff.sig_srst = module->LogicNot(NEW_ID, ff.sig_srst); + port.clk = ff.sig_clk; + port.clk_enable = true; + port.clk_polarity = ff.pol_clk; + if (ff.has_ce) + port.en = ff.sig_ce; else - { - if (find_sig_before_dff(sig_data, clk_data, clk_polarity, true) && clk_data != RTLIL::SigSpec(RTLIL::State::Sx)) - { - disconnect_dff(sig_data); - cell->setPort(ID::CLK, clk_data); - cell->setPort(ID::EN, State::S1); - cell->setPort(ID::DATA, sig_data); - cell->parameters[ID::CLK_ENABLE] = RTLIL::Const(1); - cell->parameters[ID::CLK_POLARITY] = RTLIL::Const(clk_polarity); - cell->parameters[ID::TRANSPARENT] = RTLIL::Const(0); - log("merged data $dff to cell.\n"); - return; + port.en = State::S1; + if (ff.has_arst) { + port.arst = ff.sig_arst; + port.arst_value = ff.val_arst; + } else { + port.arst = State::S0; + } + if (ff.has_srst) { + port.srst = ff.sig_srst; + port.srst_value = ff.val_srst; + port.ce_over_srst = ff.ce_over_srst; + } else { + port.srst = State::S0; + } + port.init_value = ff.val_init; + port.data = ff.sig_q; + for (int pi = 0; pi < GetSize(mem.wr_ports); pi++) { + auto &pd = portdata[pi]; + if (!pd.relevant) + continue; + if (pd.final_collision_x) { + log(" Write port %d: don't care on collision.\n", pi); + port.collision_x_mask[pi] = true; + } else if (pd.final_transparency) { + log(" Write port %d: transparent.\n", pi); + port.transparency_mask[pi] = true; + } else { + log(" Write port %d: non-transparent.\n", pi); } } + mem.emit(); + } - skip_ff_after_read_merging:; - RTLIL::SigSpec clk_addr = RTLIL::SigSpec(RTLIL::State::Sx); - RTLIL::SigSpec sig_addr = cell->getPort(ID::ADDR); - if (find_sig_before_dff(sig_addr, clk_addr, clk_polarity) && - clk_addr != RTLIL::SigSpec(RTLIL::State::Sx)) - { - cell->setPort(ID::CLK, clk_addr); - cell->setPort(ID::EN, State::S1); - cell->setPort(ID::ADDR, sig_addr); - cell->parameters[ID::CLK_ENABLE] = RTLIL::Const(1); - cell->parameters[ID::CLK_POLARITY] = RTLIL::Const(clk_polarity); - cell->parameters[ID::TRANSPARENT] = RTLIL::Const(1); - log("merged address $dff to cell.\n"); + void handle_rd_port_addr(Mem &mem, int idx) + { + auto &port = mem.rd_ports[idx]; + log("Checking read port address `%s'[%d] in module `%s': ", mem.memid.c_str(), idx, module->name.c_str()); + + FfData ff; + pool<std::pair<Cell *, int>> bits; + if (!merger.find_input_ff(port.addr, ff, bits)) { + log("no address FF found.\n"); return; } - - log("no (compatible) $dff found.\n"); + if (!ff.has_clk) { + log("address latches are not supported.\n"); + return; + } + if (ff.has_aload) { + log("address FF has async load, not supported.\n"); + return; + } + if (ff.has_sr || ff.has_arst) { + log("address FF has async set and/or reset, not supported.\n"); + return; + } + // Trick part: this transform is invalid if the initial + // value of the FF is fully-defined. However, we + // cannot simply reject FFs with any defined init bit, + // as this is often the result of merging a const bit. + if (ff.val_init.is_fully_def()) { + log("address FF has fully-defined init value, not supported.\n"); + return; + } + for (int i = 0; i < GetSize(mem.wr_ports); i++) { + auto &wport = mem.wr_ports[i]; + if (!wport.clk_enable || wport.clk != ff.sig_clk || wport.clk_polarity != ff.pol_clk) { + log("address FF clock is not compatible with write clock.\n"); + return; + } + } + // Now we're commited to merge it. + merger.mark_input_ff(bits); + // If the address FF has enable and/or sync reset, unmap it. + ff.unmap_ce_srst(); + port.clk = ff.sig_clk; + port.en = State::S1; + port.addr = ff.sig_d; + port.clk_enable = true; + port.clk_polarity = ff.pol_clk; + for (int i = 0; i < GetSize(mem.wr_ports); i++) + port.transparency_mask[i] = true; + mem.emit(); + log("merged address FF to cell.\n"); } - void run(bool flag_wr_only) + void run() { - for (auto wire : module->wires()) { - if (wire->port_output) - for (auto bit : sigmap(wire)) - sigbit_users_count[bit]++; - } - - for (auto cell : module->cells()) { - if (cell->type == ID($dff)) - dff_cells.push_back(cell); - if (cell->type == ID($mux)) { - mux_cells_a[sigmap(cell->getPort(ID::A))] = cell; - mux_cells_b[sigmap(cell->getPort(ID::B))] = cell; + std::vector<Mem> memories = Mem::get_selected_memories(module); + for (auto &mem : memories) { + QuickConeSat qcsat(modwalker); + for (int i = 0; i < GetSize(mem.rd_ports); i++) { + if (!mem.rd_ports[i].clk_enable) + handle_rd_port(mem, qcsat, i); } - if (cell->type.in(ID($not), ID($_NOT_)) || (cell->type == ID($logic_not) && GetSize(cell->getPort(ID::A)) == 1)) { - SigSpec sig_a = cell->getPort(ID::A); - SigSpec sig_y = cell->getPort(ID::Y); - if (cell->type == ID($not)) - sig_a.extend_u0(GetSize(sig_y), cell->getParam(ID::A_SIGNED).as_bool()); - if (cell->type == ID($logic_not)) - sig_y.extend_u0(1); - for (int i = 0; i < GetSize(sig_y); i++) - invbits[sig_y[i]] = sig_a[i]; + } + for (auto &mem : memories) { + for (int i = 0; i < GetSize(mem.rd_ports); i++) { + if (!mem.rd_ports[i].clk_enable) + handle_rd_port_addr(mem, i); } - for (auto &conn : cell->connections()) - if (!cell->known() || cell->input(conn.first)) - for (auto bit : sigmap(conn.second)) - sigbit_users_count[bit]++; } - - for (auto cell : module->selected_cells()) - if (cell->type == ID($memwr) && !cell->parameters[ID::CLK_ENABLE].as_bool()) - handle_wr_cell(cell); - - if (!flag_wr_only) - for (auto cell : module->selected_cells()) - if (cell->type == ID($memrd) && !cell->parameters[ID::CLK_ENABLE].as_bool()) - handle_rd_cell(cell); } }; struct MemoryDffPass : public Pass { - MemoryDffPass() : Pass("memory_dff", "merge input/output DFFs into memories") { } + MemoryDffPass() : Pass("memory_dff", "merge input/output DFFs into memory read ports") { } void help() override { // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| log("\n"); log(" memory_dff [options] [selection]\n"); log("\n"); - log("This pass detects DFFs at memory ports and merges them into the memory port.\n"); + log("This pass detects DFFs at memory read ports and merges them into the memory port.\n"); log("I.e. it consumes an asynchronous memory port and the flip-flops at its\n"); log("interface and yields a synchronous memory port.\n"); log("\n"); - log(" -nordfff\n"); - log(" do not merge registers on read ports\n"); - log("\n"); } void execute(std::vector<std::string> args, RTLIL::Design *design) override { - bool flag_wr_only = false; - - log_header(design, "Executing MEMORY_DFF pass (merging $dff cells to $memrd and $memwr).\n"); + log_header(design, "Executing MEMORY_DFF pass (merging $dff cells to $memrd).\n"); size_t argidx; for (argidx = 1; argidx < args.size(); argidx++) { - if (args[argidx] == "-nordff" || args[argidx] == "-wr_only") { - flag_wr_only = true; - continue; - } break; } extra_args(args, argidx, design); for (auto mod : design->selected_modules()) { MemoryDffWorker worker(mod); - worker.run(flag_wr_only); + worker.run(); } } } MemoryDffPass; diff --git a/passes/memory/memory_map.cc b/passes/memory/memory_map.cc index 80dd3957d..ca1ca483d 100644 --- a/passes/memory/memory_map.cc +++ b/passes/memory/memory_map.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -19,6 +19,7 @@ #include "kernel/register.h" #include "kernel/log.h" +#include "kernel/mem.h" #include <sstream> #include <set> #include <stdlib.h> @@ -33,10 +34,12 @@ struct MemoryMapWorker RTLIL::Design *design; RTLIL::Module *module; + SigMap sigmap; + FfInitVals initvals; std::map<std::pair<RTLIL::SigSpec, RTLIL::SigSpec>, RTLIL::SigBit> decoder_cache; - MemoryMapWorker(RTLIL::Design *design, RTLIL::Module *module) : design(design), module(module) {} + MemoryMapWorker(RTLIL::Design *design, RTLIL::Module *module) : design(design), module(module), sigmap(module), initvals(&sigmap, module) {} std::string map_case(std::string value) const { @@ -97,35 +100,26 @@ struct MemoryMapWorker return bit.wire; } - void handle_cell(RTLIL::Cell *cell) + void handle_memory(Mem &mem) { std::set<int> static_ports; std::map<int, RTLIL::SigSpec> static_cells_map; - int wr_ports = cell->parameters[ID::WR_PORTS].as_int(); - int rd_ports = cell->parameters[ID::RD_PORTS].as_int(); - - int mem_size = cell->parameters[ID::SIZE].as_int(); - int mem_width = cell->parameters[ID::WIDTH].as_int(); - int mem_offset = cell->parameters[ID::OFFSET].as_int(); - int mem_abits = cell->parameters[ID::ABITS].as_int(); - - SigSpec init_data = cell->getParam(ID::INIT); - init_data.extend_u0(mem_size*mem_width, true); + SigSpec init_data = mem.get_init_data(); // delete unused memory cell - if (wr_ports == 0 && rd_ports == 0) { - module->remove(cell); + if (mem.rd_ports.empty()) { + mem.remove(); return; } - // check if attributes allow us to infer FFRAM for this cell + // check if attributes allow us to infer FFRAM for this memory for (const auto &attr : attributes) { - if (cell->attributes.count(attr.first)) { - const auto &cell_attr = cell->attributes[attr.first]; + if (mem.attributes.count(attr.first)) { + const auto &cell_attr = mem.attributes[attr.first]; if (attr.second.empty()) { - log("Not mapping memory cell %s in module %s (attribute %s is set).\n", - cell->name.c_str(), module->name.c_str(), attr.first.c_str()); + log("Not mapping memory %s in module %s (attribute %s is set).\n", + mem.memid.c_str(), module->name.c_str(), attr.first.c_str()); return; } @@ -138,11 +132,11 @@ struct MemoryMapWorker } if (!found) { if (cell_attr.flags & RTLIL::CONST_FLAG_STRING) { - log("Not mapping memory cell %s in module %s (attribute %s is set to \"%s\").\n", - cell->name.c_str(), module->name.c_str(), attr.first.c_str(), cell_attr.decode_string().c_str()); + log("Not mapping memory %s in module %s (attribute %s is set to \"%s\").\n", + mem.memid.c_str(), module->name.c_str(), attr.first.c_str(), cell_attr.decode_string().c_str()); } else { - log("Not mapping memory cell %s in module %s (attribute %s is set to %d).\n", - cell->name.c_str(), module->name.c_str(), attr.first.c_str(), cell_attr.as_int()); + log("Not mapping memory %s in module %s (attribute %s is set to %d).\n", + mem.memid.c_str(), module->name.c_str(), attr.first.c_str(), cell_attr.as_int()); } return; } @@ -150,161 +144,113 @@ struct MemoryMapWorker } // all write ports must share the same clock - RTLIL::SigSpec clocks = cell->getPort(ID::WR_CLK); - RTLIL::Const clocks_pol = cell->parameters[ID::WR_CLK_POLARITY]; - RTLIL::Const clocks_en = cell->parameters[ID::WR_CLK_ENABLE]; - clocks_pol.bits.resize(wr_ports); - clocks_en.bits.resize(wr_ports); RTLIL::SigSpec refclock; - RTLIL::State refclock_pol = RTLIL::State::Sx; - for (int i = 0; i < clocks.size(); i++) { - RTLIL::SigSpec wr_en = cell->getPort(ID::WR_EN).extract(i * mem_width, mem_width); - if (wr_en.is_fully_const() && !wr_en.as_bool()) { + bool refclock_pol = false; + for (int i = 0; i < GetSize(mem.wr_ports); i++) { + auto &port = mem.wr_ports[i]; + if (port.en.is_fully_const() && !port.en.as_bool()) { static_ports.insert(i); continue; } - if (clocks_en.bits[i] != RTLIL::State::S1) { - RTLIL::SigSpec wr_addr = cell->getPort(ID::WR_ADDR).extract(i*mem_abits, mem_abits); - RTLIL::SigSpec wr_data = cell->getPort(ID::WR_DATA).extract(i*mem_width, mem_width); - if (wr_addr.is_fully_const()) { - // FIXME: Actually we should check for wr_en.is_fully_const() also and - // create a $adff cell with this ports wr_en input as reset pin when wr_en - // is not a simple static 1. - static_cells_map[wr_addr.as_int() - mem_offset] = wr_data; + if (!port.clk_enable) { + if (port.addr.is_fully_const() && port.en.is_fully_ones()) { + for (int sub = 0; sub < (1 << port.wide_log2); sub++) + static_cells_map[port.addr.as_int() + sub] = port.data.extract(sub * mem.width, mem.width); static_ports.insert(i); continue; } - log("Not mapping memory cell %s in module %s (write port %d has no clock).\n", - cell->name.c_str(), module->name.c_str(), i); + log("Not mapping memory %s in module %s (write port %d has no clock).\n", + mem.memid.c_str(), module->name.c_str(), i); return; } if (refclock.size() == 0) { - refclock = clocks.extract(i, 1); - refclock_pol = clocks_pol.bits[i]; + refclock = port.clk; + refclock_pol = port.clk_polarity; } - if (clocks.extract(i, 1) != refclock || clocks_pol.bits[i] != refclock_pol) { - log("Not mapping memory cell %s in module %s (write clock %d is incompatible with other clocks).\n", - cell->name.c_str(), module->name.c_str(), i); + if (port.clk != refclock || port.clk_polarity != refclock_pol) { + log("Not mapping memory %s in module %s (write clock %d is incompatible with other clocks).\n", + mem.memid.c_str(), module->name.c_str(), i); return; } } - log("Mapping memory cell %s in module %s:\n", cell->name.c_str(), module->name.c_str()); + log("Mapping memory %s in module %s:\n", mem.memid.c_str(), module->name.c_str()); - std::vector<RTLIL::SigSpec> data_reg_in; - std::vector<RTLIL::SigSpec> data_reg_out; + int abits = ceil_log2(mem.size); + std::vector<RTLIL::SigSpec> data_reg_in(1 << abits); + std::vector<RTLIL::SigSpec> data_reg_out(1 << abits); int count_static = 0; - for (int i = 0; i < mem_size; i++) + for (int i = 0; i < mem.size; i++) { - if (static_cells_map.count(i) > 0) + int addr = i + mem.start_offset; + int idx = addr & ((1 << abits) - 1); + if (static_cells_map.count(addr) > 0) { - data_reg_in.push_back(RTLIL::SigSpec(RTLIL::State::Sz, mem_width)); - data_reg_out.push_back(static_cells_map[i]); + data_reg_out[idx] = static_cells_map[addr]; count_static++; } else { - RTLIL::Cell *c = module->addCell(genid(cell->name, "", i), ID($dff)); - c->parameters[ID::WIDTH] = cell->parameters[ID::WIDTH]; - if (clocks_pol.bits.size() > 0) { - c->parameters[ID::CLK_POLARITY] = RTLIL::Const(clocks_pol.bits[0]); - c->setPort(ID::CLK, clocks.extract(0, 1)); + RTLIL::Cell *c = module->addCell(genid(mem.memid, "", addr), ID($dff)); + c->parameters[ID::WIDTH] = mem.width; + if (GetSize(refclock) != 0) { + c->parameters[ID::CLK_POLARITY] = RTLIL::Const(refclock_pol); + c->setPort(ID::CLK, refclock); } else { c->parameters[ID::CLK_POLARITY] = RTLIL::Const(RTLIL::State::S1); c->setPort(ID::CLK, RTLIL::SigSpec(RTLIL::State::S0)); } - RTLIL::Wire *w_in = module->addWire(genid(cell->name, "", i, "$d"), mem_width); - data_reg_in.push_back(RTLIL::SigSpec(w_in)); - c->setPort(ID::D, data_reg_in.back()); + RTLIL::Wire *w_in = module->addWire(genid(mem.memid, "", addr, "$d"), mem.width); + data_reg_in[idx] = w_in; + c->setPort(ID::D, w_in); - std::string w_out_name = stringf("%s[%d]", cell->parameters[ID::MEMID].decode_string().c_str(), i); + std::string w_out_name = stringf("%s[%d]", mem.memid.c_str(), addr); if (module->wires_.count(w_out_name) > 0) - w_out_name = genid(cell->name, "", i, "$q"); + w_out_name = genid(mem.memid, "", addr, "$q"); - RTLIL::Wire *w_out = module->addWire(w_out_name, mem_width); - SigSpec w_init = init_data.extract(i*mem_width, mem_width); + RTLIL::Wire *w_out = module->addWire(w_out_name, mem.width); + SigSpec w_init = init_data.extract(i*mem.width, mem.width); if (!w_init.is_fully_undef()) w_out->attributes[ID::init] = w_init.as_const(); - data_reg_out.push_back(RTLIL::SigSpec(w_out)); - c->setPort(ID::Q, data_reg_out.back()); + data_reg_out[idx] = w_out; + c->setPort(ID::Q, w_out); } } - log(" created %d $dff cells and %d static cells of width %d.\n", mem_size-count_static, count_static, mem_width); + log(" created %d $dff cells and %d static cells of width %d.\n", mem.size-count_static, count_static, mem.width); int count_dff = 0, count_mux = 0, count_wrmux = 0; - for (int i = 0; i < cell->parameters[ID::RD_PORTS].as_int(); i++) + for (int i = 0; i < GetSize(mem.rd_ports); i++) { - RTLIL::SigSpec rd_addr = cell->getPort(ID::RD_ADDR).extract(i*mem_abits, mem_abits); - - if (mem_offset) - rd_addr = module->Sub(NEW_ID, rd_addr, SigSpec(mem_offset, GetSize(rd_addr))); + auto &port = mem.rd_ports[i]; + if (mem.extract_rdff(i, &initvals)) + count_dff++; + RTLIL::SigSpec rd_addr = port.addr; + rd_addr.extend_u0(abits, false); std::vector<RTLIL::SigSpec> rd_signals; - rd_signals.push_back(cell->getPort(ID::RD_DATA).extract(i*mem_width, mem_width)); - - if (cell->parameters[ID::RD_CLK_ENABLE].bits[i] == RTLIL::State::S1) - { - RTLIL::Cell *dff_cell = nullptr; - - if (cell->parameters[ID::RD_TRANSPARENT].bits[i] == RTLIL::State::S1) - { - dff_cell = module->addCell(genid(cell->name, "$rdreg", i), ID($dff)); - dff_cell->parameters[ID::WIDTH] = RTLIL::Const(mem_abits); - dff_cell->parameters[ID::CLK_POLARITY] = RTLIL::Const(cell->parameters[ID::RD_CLK_POLARITY].bits[i]); - dff_cell->setPort(ID::CLK, cell->getPort(ID::RD_CLK).extract(i, 1)); - dff_cell->setPort(ID::D, rd_addr); - count_dff++; - - RTLIL::Wire *w = module->addWire(genid(cell->name, "$rdreg", i, "$q"), mem_abits); + rd_signals.push_back(port.data); - dff_cell->setPort(ID::Q, RTLIL::SigSpec(w)); - rd_addr = RTLIL::SigSpec(w); - } - else - { - dff_cell = module->addCell(genid(cell->name, "$rdreg", i), ID($dff)); - dff_cell->parameters[ID::WIDTH] = cell->parameters[ID::WIDTH]; - dff_cell->parameters[ID::CLK_POLARITY] = RTLIL::Const(cell->parameters[ID::RD_CLK_POLARITY].bits[i]); - dff_cell->setPort(ID::CLK, cell->getPort(ID::RD_CLK).extract(i, 1)); - dff_cell->setPort(ID::Q, rd_signals.back()); - count_dff++; - - RTLIL::Wire *w = module->addWire(genid(cell->name, "$rdreg", i, "$d"), mem_width); - - rd_signals.clear(); - rd_signals.push_back(RTLIL::SigSpec(w)); - dff_cell->setPort(ID::D, rd_signals.back()); - } - - SigBit en_bit = cell->getPort(ID::RD_EN).extract(i); - if (en_bit != State::S1) { - SigSpec new_d = module->Mux(genid(cell->name, "$rdenmux", i), - dff_cell->getPort(ID::Q), dff_cell->getPort(ID::D), en_bit); - dff_cell->setPort(ID::D, new_d); - } - } - - for (int j = 0; j < mem_abits; j++) + for (int j = 0; j < abits - port.wide_log2; j++) { std::vector<RTLIL::SigSpec> next_rd_signals; for (size_t k = 0; k < rd_signals.size(); k++) { - RTLIL::Cell *c = module->addCell(genid(cell->name, "$rdmux", i, "", j, "", k), ID($mux)); - c->parameters[ID::WIDTH] = cell->parameters[ID::WIDTH]; + RTLIL::Cell *c = module->addCell(genid(mem.memid, "$rdmux", i, "", j, "", k), ID($mux)); + c->parameters[ID::WIDTH] = GetSize(port.data); c->setPort(ID::Y, rd_signals[k]); - c->setPort(ID::S, rd_addr.extract(mem_abits-j-1, 1)); + c->setPort(ID::S, rd_addr.extract(abits-j-1, 1)); count_mux++; - c->setPort(ID::A, module->addWire(genid(cell->name, "$rdmux", i, "", j, "", k, "$a"), mem_width)); - c->setPort(ID::B, module->addWire(genid(cell->name, "$rdmux", i, "", j, "", k, "$b"), mem_width)); + c->setPort(ID::A, module->addWire(genid(mem.memid, "$rdmux", i, "", j, "", k, "$a"), GetSize(port.data))); + c->setPort(ID::B, module->addWire(genid(mem.memid, "$rdmux", i, "", j, "", k, "$b"), GetSize(port.data))); next_rd_signals.push_back(c->getPort(ID::A)); next_rd_signals.push_back(c->getPort(ID::B)); @@ -313,38 +259,38 @@ struct MemoryMapWorker next_rd_signals.swap(rd_signals); } - for (int j = 0; j < mem_size; j++) - module->connect(RTLIL::SigSig(rd_signals[j], data_reg_out[j])); + for (int j = 0; j < (1 << abits); j++) + if (data_reg_out[j] != SigSpec()) + module->connect(RTLIL::SigSig(rd_signals[j >> port.wide_log2].extract((j & ((1 << port.wide_log2) - 1)) * mem.width, mem.width), data_reg_out[j])); } log(" read interface: %d $dff and %d $mux cells.\n", count_dff, count_mux); - for (int i = 0; i < mem_size; i++) + for (int i = 0; i < mem.size; i++) { - if (static_cells_map.count(i) > 0) + int addr = i + mem.start_offset; + int idx = addr & ((1 << abits) - 1); + if (static_cells_map.count(addr) > 0) continue; - RTLIL::SigSpec sig = data_reg_out[i]; + RTLIL::SigSpec sig = data_reg_out[idx]; - for (int j = 0; j < cell->parameters[ID::WR_PORTS].as_int(); j++) + for (int j = 0; j < GetSize(mem.wr_ports); j++) { - RTLIL::SigSpec wr_addr = cell->getPort(ID::WR_ADDR).extract(j*mem_abits, mem_abits); - RTLIL::SigSpec wr_data = cell->getPort(ID::WR_DATA).extract(j*mem_width, mem_width); - RTLIL::SigSpec wr_en = cell->getPort(ID::WR_EN).extract(j*mem_width, mem_width); - - if (mem_offset) - wr_addr = module->Sub(NEW_ID, wr_addr, SigSpec(mem_offset, GetSize(wr_addr))); + auto &port = mem.wr_ports[j]; + RTLIL::SigSpec wr_addr = port.addr.extract_end(port.wide_log2); + RTLIL::Wire *w_seladdr = addr_decode(wr_addr, RTLIL::SigSpec(addr >> port.wide_log2, GetSize(wr_addr))); - RTLIL::Wire *w_seladdr = addr_decode(wr_addr, RTLIL::SigSpec(i, mem_abits)); + int sub = addr & ((1 << port.wide_log2) - 1); int wr_offset = 0; - while (wr_offset < wr_en.size()) + while (wr_offset < mem.width) { int wr_width = 1; - RTLIL::SigSpec wr_bit = wr_en.extract(wr_offset, 1); + RTLIL::SigSpec wr_bit = port.en.extract(wr_offset + sub * mem.width, 1); - while (wr_offset + wr_width < wr_en.size()) { - RTLIL::SigSpec next_wr_bit = wr_en.extract(wr_offset + wr_width, 1); + while (wr_offset + wr_width < mem.width) { + RTLIL::SigSpec next_wr_bit = port.en.extract(wr_offset + wr_width + sub * mem.width, 1); if (next_wr_bit != wr_bit) break; wr_width++; @@ -354,7 +300,7 @@ struct MemoryMapWorker if (wr_bit != State::S1) { - RTLIL::Cell *c = module->addCell(genid(cell->name, "$wren", i, "", j, "", wr_offset), ID($and)); + RTLIL::Cell *c = module->addCell(genid(mem.memid, "$wren", addr, "", j, "", wr_offset), ID($and)); c->parameters[ID::A_SIGNED] = RTLIL::Const(0); c->parameters[ID::B_SIGNED] = RTLIL::Const(0); c->parameters[ID::A_WIDTH] = RTLIL::Const(1); @@ -363,17 +309,17 @@ struct MemoryMapWorker c->setPort(ID::A, w); c->setPort(ID::B, wr_bit); - w = module->addWire(genid(cell->name, "$wren", i, "", j, "", wr_offset, "$y")); + w = module->addWire(genid(mem.memid, "$wren", addr, "", j, "", wr_offset, "$y")); c->setPort(ID::Y, RTLIL::SigSpec(w)); } - RTLIL::Cell *c = module->addCell(genid(cell->name, "$wrmux", i, "", j, "", wr_offset), ID($mux)); + RTLIL::Cell *c = module->addCell(genid(mem.memid, "$wrmux", addr, "", j, "", wr_offset), ID($mux)); c->parameters[ID::WIDTH] = wr_width; c->setPort(ID::A, sig.extract(wr_offset, wr_width)); - c->setPort(ID::B, wr_data.extract(wr_offset, wr_width)); + c->setPort(ID::B, port.data.extract(wr_offset + sub * mem.width, wr_width)); c->setPort(ID::S, RTLIL::SigSpec(w)); - w = module->addWire(genid(cell->name, "$wrmux", i, "", j, "", wr_offset, "$y"), wr_width); + w = module->addWire(genid(mem.memid, "$wrmux", addr, "", j, "", wr_offset, "$y"), wr_width); c->setPort(ID::Y, w); sig.replace(wr_offset, w); @@ -382,22 +328,18 @@ struct MemoryMapWorker } } - module->connect(RTLIL::SigSig(data_reg_in[i], sig)); + module->connect(RTLIL::SigSig(data_reg_in[idx], sig)); } log(" write interface: %d write mux blocks.\n", count_wrmux); - module->remove(cell); + mem.remove(); } void run() { - std::vector<RTLIL::Cell*> cells; - for (auto cell : module->selected_cells()) - if (cell->type == ID($mem)) - cells.push_back(cell); - for (auto cell : cells) - handle_cell(cell); + for (auto &mem : Mem::get_selected_memories(module)) + handle_memory(mem); } }; @@ -430,7 +372,7 @@ struct MemoryMapPass : public Pass { bool attr_icase = false; dict<RTLIL::IdString, std::vector<RTLIL::Const>> attributes; - log_header(design, "Executing MEMORY_MAP pass (converting $mem cells to logic and flip-flops).\n"); + log_header(design, "Executing MEMORY_MAP pass (converting memories to logic and flip-flops).\n"); size_t argidx; for (argidx = 1; argidx < args.size(); argidx++) diff --git a/passes/memory/memory_memx.cc b/passes/memory/memory_memx.cc index 02e00cf30..7edc26caa 100644 --- a/passes/memory/memory_memx.cc +++ b/passes/memory/memory_memx.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,11 +17,8 @@ * */ -#include "kernel/register.h" -#include "kernel/log.h" -#include <sstream> -#include <set> -#include <stdlib.h> +#include "kernel/yosys.h" +#include "kernel/mem.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN @@ -38,53 +35,45 @@ struct MemoryMemxPass : public Pass { log("behavior for out-of-bounds memory reads and writes.\n"); log("\n"); } + + SigSpec make_addr_check(Mem &mem, SigSpec addr) { + int start_addr = mem.start_offset; + int end_addr = mem.start_offset + mem.size; + + addr.extend_u0(32); + + SigSpec res = mem.module->Nex(NEW_ID, mem.module->ReduceXor(NEW_ID, addr), mem.module->ReduceXor(NEW_ID, {addr, State::S1})); + if (start_addr != 0) + res = mem.module->LogicAnd(NEW_ID, res, mem.module->Ge(NEW_ID, addr, start_addr)); + res = mem.module->LogicAnd(NEW_ID, res, mem.module->Lt(NEW_ID, addr, end_addr)); + return res; + } + void execute(std::vector<std::string> args, RTLIL::Design *design) override { log_header(design, "Executing MEMORY_MEMX pass (converting $mem cells to logic and flip-flops).\n"); extra_args(args, 1, design); for (auto module : design->selected_modules()) + for (auto &mem : Mem::get_selected_memories(module)) { - vector<Cell*> mem_port_cells; - - for (auto cell : module->selected_cells()) - if (cell->type.in(ID($memrd), ID($memwr))) - mem_port_cells.push_back(cell); - - for (auto cell : mem_port_cells) + for (auto &port : mem.rd_ports) { - IdString memid = cell->getParam(ID::MEMID).decode_string(); - RTLIL::Memory *mem = module->memories.at(memid); + if (port.clk_enable) + log_error("Memory %s.%s has a synchronous read port. Synchronous read ports are not supported by memory_memx!\n", + log_id(module), log_id(mem.memid)); - int lowest_addr = mem->start_offset; - int highest_addr = mem->start_offset + mem->size - 1; - - SigSpec addr = cell->getPort(ID::ADDR); - addr.extend_u0(32); - - SigSpec addr_ok = module->Nex(NEW_ID, module->ReduceXor(NEW_ID, addr), module->ReduceXor(NEW_ID, {addr, State::S1})); - if (lowest_addr != 0) - addr_ok = module->LogicAnd(NEW_ID, addr_ok, module->Ge(NEW_ID, addr, lowest_addr)); - addr_ok = module->LogicAnd(NEW_ID, addr_ok, module->Le(NEW_ID, addr, highest_addr)); - - if (cell->type == ID($memrd)) - { - if (cell->getParam(ID::CLK_ENABLE).as_bool()) - log_error("Cell %s.%s (%s) has an enabled clock. Clocked $memrd cells are not supported by memory_memx!\n", - log_id(module), log_id(cell), log_id(cell->type)); - - SigSpec rdata = cell->getPort(ID::DATA); - Wire *raw_rdata = module->addWire(NEW_ID, GetSize(rdata)); - module->addMux(NEW_ID, SigSpec(State::Sx, GetSize(rdata)), raw_rdata, addr_ok, rdata); - cell->setPort(ID::DATA, raw_rdata); - } + SigSpec addr_ok = make_addr_check(mem, port.addr); + Wire *raw_rdata = module->addWire(NEW_ID, GetSize(port.data)); + module->addMux(NEW_ID, SigSpec(State::Sx, GetSize(port.data)), raw_rdata, addr_ok, port.data); + port.data = raw_rdata; + } - if (cell->type == ID($memwr)) - { - SigSpec en = cell->getPort(ID::EN); - en = module->And(NEW_ID, en, addr_ok.repeat(GetSize(en))); - cell->setPort(ID::EN, en); - } + for (auto &port : mem.wr_ports) { + SigSpec addr_ok = make_addr_check(mem, port.addr); + port.en = module->And(NEW_ID, port.en, addr_ok.repeat(GetSize(port.en))); } + + mem.emit(); } } } MemoryMemxPass; diff --git a/passes/memory/memory_narrow.cc b/passes/memory/memory_narrow.cc new file mode 100644 index 000000000..cf5e43465 --- /dev/null +++ b/passes/memory/memory_narrow.cc @@ -0,0 +1,67 @@ +/* + * yosys -- Yosys Open SYnthesis Suite + * + * Copyright (C) 2020 Marcelina Kościelnicka <mwk@0x04.net> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + */ + +#include "kernel/yosys.h" +#include "kernel/sigtools.h" +#include "kernel/mem.h" + +USING_YOSYS_NAMESPACE +PRIVATE_NAMESPACE_BEGIN + +struct MemoryNarrowPass : public Pass { + MemoryNarrowPass() : Pass("memory_narrow", "split up wide memory ports") { } + void help() override + { + // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| + log("\n"); + log(" memory_narrow [options] [selection]\n"); + log("\n"); + log("This pass splits up wide memory ports into several narrow ports.\n"); + log("\n"); + } + void execute(std::vector<std::string> args, RTLIL::Design *design) override + { + log_header(design, "Executing MEMORY_NARROW pass (splitting up wide memory ports).\n"); + + size_t argidx; + for (argidx = 1; argidx < args.size(); argidx++) { + break; + } + extra_args(args, argidx, design); + + for (auto module : design->selected_modules()) { + for (auto &mem : Mem::get_selected_memories(module)) + { + bool wide = false; + for (auto &port : mem.rd_ports) + if (port.wide_log2) + wide = true; + for (auto &port : mem.wr_ports) + if (port.wide_log2) + wide = true; + if (wide) { + mem.narrow(); + mem.emit(); + } + } + } + } +} MemoryNarrowPass; + +PRIVATE_NAMESPACE_END diff --git a/passes/memory/memory_nordff.cc b/passes/memory/memory_nordff.cc index 07bbd9fe8..3253c8f60 100644 --- a/passes/memory/memory_nordff.cc +++ b/passes/memory/memory_nordff.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -19,6 +19,7 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" +#include "kernel/mem.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN @@ -32,12 +33,12 @@ struct MemoryNordffPass : public Pass { log(" memory_nordff [options] [selection]\n"); log("\n"); log("This pass extracts FFs from memory read ports. This results in a netlist\n"); - log("similar to what one would get from calling memory_dff with -nordff.\n"); + log("similar to what one would get from not calling memory_dff.\n"); log("\n"); } void execute(std::vector<std::string> args, RTLIL::Design *design) override { - log_header(design, "Executing MEMORY_NORDFF pass (extracting $dff cells from $mem).\n"); + log_header(design, "Executing MEMORY_NORDFF pass (extracting $dff cells from memories).\n"); size_t argidx; for (argidx = 1; argidx < args.size(); argidx++) { @@ -50,70 +51,22 @@ struct MemoryNordffPass : public Pass { extra_args(args, argidx, design); for (auto module : design->selected_modules()) - for (auto cell : vector<Cell*>(module->selected_cells())) { - if (cell->type != ID($mem)) - continue; - - int rd_ports = cell->getParam(ID::RD_PORTS).as_int(); - int abits = cell->getParam(ID::ABITS).as_int(); - int width = cell->getParam(ID::WIDTH).as_int(); - - SigSpec rd_addr = cell->getPort(ID::RD_ADDR); - SigSpec rd_data = cell->getPort(ID::RD_DATA); - SigSpec rd_clk = cell->getPort(ID::RD_CLK); - SigSpec rd_en = cell->getPort(ID::RD_EN); - Const rd_clk_enable = cell->getParam(ID::RD_CLK_ENABLE); - Const rd_clk_polarity = cell->getParam(ID::RD_CLK_POLARITY); - - for (int i = 0; i < rd_ports; i++) + SigMap sigmap(module); + FfInitVals initvals(&sigmap, module); + for (auto &mem : Mem::get_selected_memories(module)) { - bool clk_enable = rd_clk_enable[i] == State::S1; - - if (clk_enable) - { - bool clk_polarity = cell->getParam(ID::RD_CLK_POLARITY)[i] == State::S1; - bool transparent = cell->getParam(ID::RD_TRANSPARENT)[i] == State::S1; - - SigSpec clk = cell->getPort(ID::RD_CLK)[i] ; - SigSpec en = cell->getPort(ID::RD_EN)[i]; - Cell *c; - - if (transparent) - { - SigSpec sig_q = module->addWire(NEW_ID, abits); - SigSpec sig_d = rd_addr.extract(abits * i, abits); - rd_addr.replace(abits * i, sig_q); - if (en != State::S1) - sig_d = module->Mux(NEW_ID, sig_q, sig_d, en); - c = module->addDff(NEW_ID, clk, sig_d, sig_q, clk_polarity); - } - else - { - SigSpec sig_d = module->addWire(NEW_ID, width); - SigSpec sig_q = rd_data.extract(width * i, width); - rd_data.replace(width *i, sig_d); - if (en != State::S1) - sig_d = module->Mux(NEW_ID, sig_q, sig_d, en); - c = module->addDff(NEW_ID, clk, sig_d, sig_q, clk_polarity); + bool changed = false; + for (int i = 0; i < GetSize(mem.rd_ports); i++) { + if (mem.rd_ports[i].clk_enable) { + mem.extract_rdff(i, &initvals); + changed = true; } - - log("Extracted %s FF from read port %d of %s.%s: %s\n", transparent ? "addr" : "data", - i, log_id(module), log_id(cell), log_id(c)); } - rd_en[i] = State::S1; - rd_clk[i] = State::S0; - rd_clk_enable[i] = State::S0; - rd_clk_polarity[i] = State::S1; + if (changed) + mem.emit(); } - - cell->setPort(ID::RD_ADDR, rd_addr); - cell->setPort(ID::RD_DATA, rd_data); - cell->setPort(ID::RD_CLK, rd_clk); - cell->setPort(ID::RD_EN, rd_en); - cell->setParam(ID::RD_CLK_ENABLE, rd_clk_enable); - cell->setParam(ID::RD_CLK_POLARITY, rd_clk_polarity); } } } MemoryNordffPass; diff --git a/passes/memory/memory_share.cc b/passes/memory/memory_share.cc index 7315aeae1..ceea725d8 100644 --- a/passes/memory/memory_share.cc +++ b/passes/memory/memory_share.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,467 +18,260 @@ */ #include "kernel/yosys.h" -#include "kernel/satgen.h" +#include "kernel/qcsat.h" #include "kernel/sigtools.h" #include "kernel/modtools.h" +#include "kernel/mem.h" +#include "kernel/ffinit.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN -bool memcells_cmp(RTLIL::Cell *a, RTLIL::Cell *b) -{ - if (a->type == ID($memrd) && b->type == ID($memrd)) - return a->name < b->name; - if (a->type == ID($memrd) || b->type == ID($memrd)) - return (a->type == ID($memrd)) < (b->type == ID($memrd)); - return a->parameters.at(ID::PRIORITY).as_int() < b->parameters.at(ID::PRIORITY).as_int(); -} - struct MemoryShareWorker { RTLIL::Design *design; RTLIL::Module *module; SigMap sigmap, sigmap_xmux; ModWalker modwalker; - CellTypes cone_ct; - - std::map<RTLIL::SigBit, std::pair<RTLIL::Cell*, int>> sig_to_mux; - std::map<pair<std::set<std::map<SigBit, bool>>, SigBit>, SigBit> conditions_logic_cache; - - - // ----------------------------------------------------------------- - // Converting feedbacks to async read ports to proper enable signals - // ----------------------------------------------------------------- - - bool find_data_feedback(const std::set<RTLIL::SigBit> &async_rd_bits, RTLIL::SigBit sig, - std::map<RTLIL::SigBit, bool> &state, std::set<std::map<RTLIL::SigBit, bool>> &conditions) - { - if (async_rd_bits.count(sig)) { - conditions.insert(state); - return true; - } - - if (sig_to_mux.count(sig) == 0) - return false; - - RTLIL::Cell *cell = sig_to_mux.at(sig).first; - int bit_idx = sig_to_mux.at(sig).second; - - std::vector<RTLIL::SigBit> sig_a = sigmap(cell->getPort(ID::A)); - std::vector<RTLIL::SigBit> sig_b = sigmap(cell->getPort(ID::B)); - std::vector<RTLIL::SigBit> sig_s = sigmap(cell->getPort(ID::S)); - std::vector<RTLIL::SigBit> sig_y = sigmap(cell->getPort(ID::Y)); - log_assert(sig_y.at(bit_idx) == sig); - - for (int i = 0; i < int(sig_s.size()); i++) - if (state.count(sig_s[i]) && state.at(sig_s[i]) == true) { - if (find_data_feedback(async_rd_bits, sig_b.at(bit_idx + i*sig_y.size()), state, conditions)) { - RTLIL::SigSpec new_b = cell->getPort(ID::B); - new_b.replace(bit_idx + i*sig_y.size(), RTLIL::State::Sx); - cell->setPort(ID::B, new_b); - } + FfInitVals initvals; + bool flag_widen; + bool flag_sat; + + // -------------------------------------------------- + // Consolidate read ports that read the same address + // (or close enough to be merged to wide ports) + // -------------------------------------------------- + + // A simple function to detect ports that couldn't possibly collide + // because of opposite const address bits (simplistic, but enough + // to fix problems with inferring wide ports). + bool rdwr_can_collide(Mem &mem, int ridx, int widx) { + auto &rport = mem.rd_ports[ridx]; + auto &wport = mem.wr_ports[widx]; + for (int i = std::max(rport.wide_log2, wport.wide_log2); i < GetSize(rport.addr) && i < GetSize(wport.addr); i++) { + if (rport.addr[i] == State::S1 && wport.addr[i] == State::S0) + return false; + if (rport.addr[i] == State::S0 && wport.addr[i] == State::S1) return false; - } - - - for (int i = 0; i < int(sig_s.size()); i++) - { - if (state.count(sig_s[i]) && state.at(sig_s[i]) == false) - continue; - - std::map<RTLIL::SigBit, bool> new_state = state; - new_state[sig_s[i]] = true; - - if (find_data_feedback(async_rd_bits, sig_b.at(bit_idx + i*sig_y.size()), new_state, conditions)) { - RTLIL::SigSpec new_b = cell->getPort(ID::B); - new_b.replace(bit_idx + i*sig_y.size(), RTLIL::State::Sx); - cell->setPort(ID::B, new_b); - } - } - - std::map<RTLIL::SigBit, bool> new_state = state; - for (int i = 0; i < int(sig_s.size()); i++) - new_state[sig_s[i]] = false; - - if (find_data_feedback(async_rd_bits, sig_a.at(bit_idx), new_state, conditions)) { - RTLIL::SigSpec new_a = cell->getPort(ID::A); - new_a.replace(bit_idx, RTLIL::State::Sx); - cell->setPort(ID::A, new_a); } - - return false; + return true; } - RTLIL::SigBit conditions_to_logic(std::set<std::map<RTLIL::SigBit, bool>> &conditions, SigBit olden, int &created_conditions) - { - auto key = make_pair(conditions, olden); - - if (conditions_logic_cache.count(key)) - return conditions_logic_cache.at(key); - - RTLIL::SigSpec terms; - for (auto &cond : conditions) { - RTLIL::SigSpec sig1, sig2; - for (auto &it : cond) { - sig1.append(it.first); - sig2.append(it.second ? RTLIL::State::S1 : RTLIL::State::S0); - } - terms.append(module->Ne(NEW_ID, sig1, sig2)); - created_conditions++; + bool merge_rst_value(Mem &mem, Const &res, int wide_log2, const Const &src1, int sub1, const Const &src2, int sub2) { + res = Const(State::Sx, mem.width << wide_log2); + for (int i = 0; i < GetSize(src1); i++) + res[i + sub1 * mem.width] = src1[i]; + for (int i = 0; i < GetSize(src2); i++) { + if (src2[i] == State::Sx) + continue; + auto &dst = res[i + sub2 * mem.width]; + if (dst == src2[i]) + continue; + if (dst != State::Sx) + return false; + dst = src2[i]; } - - if (olden.wire != nullptr || olden != State::S1) - terms.append(olden); - - if (GetSize(terms) == 0) - terms = State::S1; - - if (GetSize(terms) > 1) - terms = module->ReduceAnd(NEW_ID, terms); - - return conditions_logic_cache[key] = terms; + return true; } - void translate_rd_feedback_to_en(std::string memid, std::vector<RTLIL::Cell*> &rd_ports, std::vector<RTLIL::Cell*> &wr_ports) + bool consolidate_rd_by_addr(Mem &mem) { - std::map<RTLIL::SigSpec, std::vector<std::set<RTLIL::SigBit>>> async_rd_bits; - std::map<RTLIL::SigBit, std::set<RTLIL::SigBit>> muxtree_upstream_map; - std::set<RTLIL::SigBit> non_feedback_nets; - - for (auto wire : module->wires()) - if (wire->port_output) { - std::vector<RTLIL::SigBit> bits = sigmap(wire); - non_feedback_nets.insert(bits.begin(), bits.end()); - } - - for (auto cell : module->cells()) - { - bool ignore_data_port = false; - - if (cell->type.in(ID($mux), ID($pmux))) - { - std::vector<RTLIL::SigBit> sig_a = sigmap(cell->getPort(ID::A)); - std::vector<RTLIL::SigBit> sig_b = sigmap(cell->getPort(ID::B)); - std::vector<RTLIL::SigBit> sig_s = sigmap(cell->getPort(ID::S)); - std::vector<RTLIL::SigBit> sig_y = sigmap(cell->getPort(ID::Y)); - - non_feedback_nets.insert(sig_s.begin(), sig_s.end()); + if (GetSize(mem.rd_ports) <= 1) + return false; - for (int i = 0; i < int(sig_y.size()); i++) { - muxtree_upstream_map[sig_y[i]].insert(sig_a[i]); - for (int j = 0; j < int(sig_s.size()); j++) - muxtree_upstream_map[sig_y[i]].insert(sig_b[i + j*sig_y.size()]); - } + log("Consolidating read ports of memory %s.%s by address:\n", log_id(module), log_id(mem.memid)); + bool changed = false; + for (int i = 0; i < GetSize(mem.rd_ports); i++) + { + auto &port1 = mem.rd_ports[i]; + if (port1.removed) continue; - } - - if (cell->type.in(ID($memwr), ID($memrd)) && - cell->parameters.at(ID::MEMID).decode_string() == memid) - ignore_data_port = true; - - for (auto conn : cell->connections()) + for (int j = i + 1; j < GetSize(mem.rd_ports); j++) { - if (ignore_data_port && conn.first == ID::DATA) + auto &port2 = mem.rd_ports[j]; + if (port2.removed) continue; - std::vector<RTLIL::SigBit> bits = sigmap(conn.second); - non_feedback_nets.insert(bits.begin(), bits.end()); - } - } - - std::set<RTLIL::SigBit> expand_non_feedback_nets = non_feedback_nets; - while (!expand_non_feedback_nets.empty()) - { - std::set<RTLIL::SigBit> new_expand_non_feedback_nets; - - for (auto &bit : expand_non_feedback_nets) - if (muxtree_upstream_map.count(bit)) - for (auto &new_bit : muxtree_upstream_map.at(bit)) - if (!non_feedback_nets.count(new_bit)) { - non_feedback_nets.insert(new_bit); - new_expand_non_feedback_nets.insert(new_bit); - } - - expand_non_feedback_nets.swap(new_expand_non_feedback_nets); - } - - for (auto cell : rd_ports) - { - if (cell->parameters.at(ID::CLK_ENABLE).as_bool()) - continue; - - RTLIL::SigSpec sig_addr = sigmap(cell->getPort(ID::ADDR)); - std::vector<RTLIL::SigBit> sig_data = sigmap(cell->getPort(ID::DATA)); - - for (int i = 0; i < int(sig_data.size()); i++) - if (non_feedback_nets.count(sig_data[i])) - goto not_pure_feedback_port; - - async_rd_bits[sig_addr].resize(max(async_rd_bits.size(), sig_data.size())); - for (int i = 0; i < int(sig_data.size()); i++) - async_rd_bits[sig_addr][i].insert(sig_data[i]); - - not_pure_feedback_port:; - } - - if (async_rd_bits.empty()) - return; - - log("Populating enable bits on write ports of memory %s.%s with aync read feedback:\n", log_id(module), log_id(memid)); - - for (auto cell : wr_ports) - { - RTLIL::SigSpec sig_addr = sigmap_xmux(cell->getPort(ID::ADDR)); - if (!async_rd_bits.count(sig_addr)) - continue; - - log(" Analyzing write port %s.\n", log_id(cell)); - - std::vector<RTLIL::SigBit> cell_data = cell->getPort(ID::DATA); - std::vector<RTLIL::SigBit> cell_en = cell->getPort(ID::EN); - - int created_conditions = 0; - for (int i = 0; i < int(cell_data.size()); i++) - if (cell_en[i] != RTLIL::SigBit(RTLIL::State::S0)) + if (port1.clk_enable != port2.clk_enable) + continue; + if (port1.clk_enable) { + if (port1.clk != port2.clk) + continue; + if (port1.clk_polarity != port2.clk_polarity) + continue; + } + if (port1.en != port2.en) + continue; + if (port1.arst != port2.arst) + continue; + if (port1.srst != port2.srst) + continue; + if (port1.ce_over_srst != port2.ce_over_srst) + continue; + // If the width of the ports doesn't match, they can still be + // merged by widening the narrow one. Check if the conditions + // hold for that. + int wide_log2 = std::max(port1.wide_log2, port2.wide_log2); + SigSpec addr1 = sigmap_xmux(port1.addr); + SigSpec addr2 = sigmap_xmux(port2.addr); + if (GetSize(addr1) <= wide_log2) + continue; + if (GetSize(addr2) <= wide_log2) + continue; + if (!addr1.extract(0, wide_log2).is_fully_const()) + continue; + if (!addr2.extract(0, wide_log2).is_fully_const()) + continue; + if (addr1.extract_end(wide_log2) != addr2.extract_end(wide_log2)) { + // Incompatible addresses after widening. Last chance — widen both + // ports by one more bit to merge them. + if (!flag_widen) + continue; + wide_log2++; + if (addr1.extract_end(wide_log2) != addr2.extract_end(wide_log2)) + continue; + if (!addr1.extract(0, wide_log2).is_fully_const()) + continue; + if (!addr2.extract(0, wide_log2).is_fully_const()) + continue; + } + // Combine init/reset values. + SigSpec sub1_c = port1.addr.extract(0, wide_log2); + log_assert(sub1_c.is_fully_const()); + int sub1 = sub1_c.as_int(); + SigSpec sub2_c = port2.addr.extract(0, wide_log2); + log_assert(sub2_c.is_fully_const()); + int sub2 = sub2_c.as_int(); + Const init_value, arst_value, srst_value; + if (!merge_rst_value(mem, init_value, wide_log2, port1.init_value, sub1, port2.init_value, sub2)) + continue; + if (!merge_rst_value(mem, arst_value, wide_log2, port1.arst_value, sub1, port2.arst_value, sub2)) + continue; + if (!merge_rst_value(mem, srst_value, wide_log2, port1.srst_value, sub1, port2.srst_value, sub2)) + continue; + // At this point we are committed to the merge. { - std::map<RTLIL::SigBit, bool> state; - std::set<std::map<RTLIL::SigBit, bool>> conditions; - - find_data_feedback(async_rd_bits.at(sig_addr).at(i), cell_data[i], state, conditions); - cell_en[i] = conditions_to_logic(conditions, cell_en[i], created_conditions); + log(" Merging ports %d, %d (address %s).\n", i, j, log_signal(port1.addr)); + port1.addr = addr1; + port2.addr = addr2; + mem.prepare_rd_merge(i, j, &initvals); + mem.widen_prep(wide_log2); + SigSpec new_data = module->addWire(NEW_ID, mem.width << wide_log2); + module->connect(port1.data, new_data.extract(sub1 * mem.width, mem.width << port1.wide_log2)); + module->connect(port2.data, new_data.extract(sub2 * mem.width, mem.width << port2.wide_log2)); + for (int k = 0; k < wide_log2; k++) + port1.addr[k] = State::S0; + port1.init_value = init_value; + port1.arst_value = arst_value; + port1.srst_value = srst_value; + port1.wide_log2 = wide_log2; + port1.data = new_data; + port2.removed = true; + changed = true; } - - if (created_conditions) { - log(" Added enable logic for %d different cases.\n", created_conditions); - cell->setPort(ID::EN, cell_en); } } + + if (changed) + mem.emit(); + + return changed; } // ------------------------------------------------------ // Consolidate write ports that write to the same address + // (or close enough to be merged to wide ports) // ------------------------------------------------------ - RTLIL::SigSpec mask_en_naive(RTLIL::SigSpec do_mask, RTLIL::SigSpec bits, RTLIL::SigSpec mask_bits) + bool consolidate_wr_by_addr(Mem &mem) { - // this is the naive version of the function that does not care about grouping the EN bits. - - RTLIL::SigSpec inv_mask_bits = module->Not(NEW_ID, mask_bits); - RTLIL::SigSpec inv_mask_bits_filtered = module->Mux(NEW_ID, RTLIL::SigSpec(RTLIL::State::S1, bits.size()), inv_mask_bits, do_mask); - RTLIL::SigSpec result = module->And(NEW_ID, inv_mask_bits_filtered, bits); - return result; - } - - RTLIL::SigSpec mask_en_grouped(RTLIL::SigSpec do_mask, RTLIL::SigSpec bits, RTLIL::SigSpec mask_bits) - { - // this version of the function preserves the bit grouping in the EN bits. - - std::vector<RTLIL::SigBit> v_bits = bits; - std::vector<RTLIL::SigBit> v_mask_bits = mask_bits; - - std::map<std::pair<RTLIL::SigBit, RTLIL::SigBit>, std::pair<int, std::vector<int>>> groups; - RTLIL::SigSpec grouped_bits, grouped_mask_bits; - - for (int i = 0; i < bits.size(); i++) { - std::pair<RTLIL::SigBit, RTLIL::SigBit> key(v_bits[i], v_mask_bits[i]); - if (groups.count(key) == 0) { - groups[key].first = grouped_bits.size(); - grouped_bits.append(v_bits[i]); - grouped_mask_bits.append(v_mask_bits[i]); - } - groups[key].second.push_back(i); - } - - std::vector<RTLIL::SigBit> grouped_result = mask_en_naive(do_mask, grouped_bits, grouped_mask_bits); - RTLIL::SigSpec result; - - for (int i = 0; i < bits.size(); i++) { - std::pair<RTLIL::SigBit, RTLIL::SigBit> key(v_bits[i], v_mask_bits[i]); - result.append(grouped_result.at(groups.at(key).first)); - } - - return result; - } - - void merge_en_data(RTLIL::SigSpec &merged_en, RTLIL::SigSpec &merged_data, RTLIL::SigSpec next_en, RTLIL::SigSpec next_data) - { - std::vector<RTLIL::SigBit> v_old_en = merged_en; - std::vector<RTLIL::SigBit> v_next_en = next_en; - - // The new merged_en signal is just the old merged_en signal and next_en OR'ed together. - // But of course we need to preserve the bit grouping.. - - std::map<std::pair<RTLIL::SigBit, RTLIL::SigBit>, int> groups; - std::vector<RTLIL::SigBit> grouped_old_en, grouped_next_en; - RTLIL::SigSpec new_merged_en; - - for (int i = 0; i < int(v_old_en.size()); i++) { - std::pair<RTLIL::SigBit, RTLIL::SigBit> key(v_old_en[i], v_next_en[i]); - if (groups.count(key) == 0) { - groups[key] = grouped_old_en.size(); - grouped_old_en.push_back(key.first); - grouped_next_en.push_back(key.second); - } - } - - std::vector<RTLIL::SigBit> grouped_new_en = module->Or(NEW_ID, grouped_old_en, grouped_next_en); - - for (int i = 0; i < int(v_old_en.size()); i++) { - std::pair<RTLIL::SigBit, RTLIL::SigBit> key(v_old_en[i], v_next_en[i]); - new_merged_en.append(grouped_new_en.at(groups.at(key))); - } - - // Create the new merged_data signal. - - RTLIL::SigSpec new_merged_data(RTLIL::State::Sx, merged_data.size()); - - RTLIL::SigSpec old_data_set = module->And(NEW_ID, merged_en, merged_data); - RTLIL::SigSpec old_data_unset = module->And(NEW_ID, merged_en, module->Not(NEW_ID, merged_data)); - - RTLIL::SigSpec new_data_set = module->And(NEW_ID, next_en, next_data); - RTLIL::SigSpec new_data_unset = module->And(NEW_ID, next_en, module->Not(NEW_ID, next_data)); - - new_merged_data = module->Or(NEW_ID, new_merged_data, old_data_set); - new_merged_data = module->And(NEW_ID, new_merged_data, module->Not(NEW_ID, old_data_unset)); - - new_merged_data = module->Or(NEW_ID, new_merged_data, new_data_set); - new_merged_data = module->And(NEW_ID, new_merged_data, module->Not(NEW_ID, new_data_unset)); - - // Update merged_* signals - - merged_en = new_merged_en; - merged_data = new_merged_data; - } - - void consolidate_wr_by_addr(std::string memid, std::vector<RTLIL::Cell*> &wr_ports) - { - if (wr_ports.size() <= 1) - return; - - log("Consolidating write ports of memory %s.%s by address:\n", log_id(module), log_id(memid)); - - std::map<RTLIL::SigSpec, int> last_port_by_addr; - std::vector<std::vector<bool>> active_bits_on_port; + if (GetSize(mem.wr_ports) <= 1) + return false; - bool cache_clk_enable = false; - bool cache_clk_polarity = false; - RTLIL::SigSpec cache_clk; + log("Consolidating write ports of memory %s.%s by address:\n", log_id(module), log_id(mem.memid)); - for (int i = 0; i < int(wr_ports.size()); i++) + bool changed = false; + for (int i = 0; i < GetSize(mem.wr_ports); i++) { - RTLIL::Cell *cell = wr_ports.at(i); - RTLIL::SigSpec addr = sigmap_xmux(cell->getPort(ID::ADDR)); - - if (cell->parameters.at(ID::CLK_ENABLE).as_bool() != cache_clk_enable || - (cache_clk_enable && (sigmap(cell->getPort(ID::CLK)) != cache_clk || - cell->parameters.at(ID::CLK_POLARITY).as_bool() != cache_clk_polarity))) - { - cache_clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool(); - cache_clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool(); - cache_clk = sigmap(cell->getPort(ID::CLK)); - last_port_by_addr.clear(); - - if (cache_clk_enable) - log(" New clock domain: %s %s\n", cache_clk_polarity ? "posedge" : "negedge", log_signal(cache_clk)); - else - log(" New clock domain: unclocked\n"); - } - - log(" Port %d (%s) has addr %s.\n", i, log_id(cell), log_signal(addr)); - - log(" Active bits: "); - std::vector<RTLIL::SigBit> en_bits = sigmap(cell->getPort(ID::EN)); - active_bits_on_port.push_back(std::vector<bool>(en_bits.size())); - for (int k = int(en_bits.size())-1; k >= 0; k--) { - active_bits_on_port[i][k] = en_bits[k].wire != NULL || en_bits[k].data != RTLIL::State::S0; - log("%c", active_bits_on_port[i][k] ? '1' : '0'); - } - log("\n"); - - if (last_port_by_addr.count(addr)) + auto &port1 = mem.wr_ports[i]; + if (port1.removed) + continue; + if (!port1.clk_enable) + continue; + for (int j = i + 1; j < GetSize(mem.wr_ports); j++) { - int last_i = last_port_by_addr.at(addr); - log(" Merging port %d into this one.\n", last_i); - - bool found_overlapping_bits = false; - for (int k = 0; k < int(en_bits.size()); k++) { - if (active_bits_on_port[i][k] && active_bits_on_port[last_i][k]) - found_overlapping_bits = true; - active_bits_on_port[i][k] = active_bits_on_port[i][k] || active_bits_on_port[last_i][k]; - } - - // Force this ports addr input to addr directly (skip don't care muxes) - - cell->setPort(ID::ADDR, addr); - - // If any of the ports between `last_i' and `i' write to the same address, this - // will have priority over whatever `last_i` wrote. So we need to revisit those - // ports and mask the EN bits accordingly. - - RTLIL::SigSpec merged_en = sigmap(wr_ports[last_i]->getPort(ID::EN)); - - for (int j = last_i+1; j < i; j++) - { - if (wr_ports[j] == NULL) + auto &port2 = mem.wr_ports[j]; + if (port2.removed) + continue; + if (!port2.clk_enable) + continue; + if (port1.clk != port2.clk) + continue; + if (port1.clk_polarity != port2.clk_polarity) + continue; + // If the width of the ports doesn't match, they can still be + // merged by widening the narrow one. Check if the conditions + // hold for that. + int wide_log2 = std::max(port1.wide_log2, port2.wide_log2); + SigSpec addr1 = sigmap_xmux(port1.addr); + SigSpec addr2 = sigmap_xmux(port2.addr); + if (GetSize(addr1) <= wide_log2) + continue; + if (GetSize(addr2) <= wide_log2) + continue; + if (!addr1.extract(0, wide_log2).is_fully_const()) + continue; + if (!addr2.extract(0, wide_log2).is_fully_const()) + continue; + if (addr1.extract_end(wide_log2) != addr2.extract_end(wide_log2)) { + // Incompatible addresses after widening. Last chance — widen both + // ports by one more bit to merge them. + if (!flag_widen) + continue; + wide_log2++; + if (addr1.extract_end(wide_log2) != addr2.extract_end(wide_log2)) + continue; + if (!addr1.extract(0, wide_log2).is_fully_const()) + continue; + if (!addr2.extract(0, wide_log2).is_fully_const()) continue; - - for (int k = 0; k < int(en_bits.size()); k++) - if (active_bits_on_port[i][k] && active_bits_on_port[j][k]) - goto found_overlapping_bits_i_j; - - if (0) { - found_overlapping_bits_i_j: - log(" Creating collosion-detect logic for port %d.\n", j); - RTLIL::SigSpec is_same_addr = module->addWire(NEW_ID); - module->addEq(NEW_ID, addr, wr_ports[j]->getPort(ID::ADDR), is_same_addr); - merged_en = mask_en_grouped(is_same_addr, merged_en, sigmap(wr_ports[j]->getPort(ID::EN))); - } } - - // Then we need to merge the (masked) EN and the DATA signals. - - RTLIL::SigSpec merged_data = wr_ports[last_i]->getPort(ID::DATA); - if (found_overlapping_bits) { - log(" Creating logic for merging DATA and EN ports.\n"); - merge_en_data(merged_en, merged_data, sigmap(cell->getPort(ID::EN)), sigmap(cell->getPort(ID::DATA))); - } else { - RTLIL::SigSpec cell_en = sigmap(cell->getPort(ID::EN)); - RTLIL::SigSpec cell_data = sigmap(cell->getPort(ID::DATA)); - for (int k = 0; k < int(en_bits.size()); k++) - if (!active_bits_on_port[last_i][k]) { - merged_en.replace(k, cell_en.extract(k, 1)); - merged_data.replace(k, cell_data.extract(k, 1)); - } + log(" Merging ports %d, %d (address %s).\n", i, j, log_signal(addr1)); + port1.addr = addr1; + port2.addr = addr2; + mem.prepare_wr_merge(i, j, &initvals); + mem.widen_wr_port(i, wide_log2); + mem.widen_wr_port(j, wide_log2); + int pos = 0; + while (pos < GetSize(port1.data)) { + int epos = pos; + while (epos < GetSize(port1.data) && port1.en[epos] == port1.en[pos] && port2.en[epos] == port2.en[pos]) + epos++; + int width = epos - pos; + SigBit new_en; + if (port2.en[pos] == State::S0) { + new_en = port1.en[pos]; + } else if (port1.en[pos] == State::S0) { + port1.data.replace(pos, port2.data.extract(pos, width)); + new_en = port2.en[pos]; + } else { + port1.data.replace(pos, module->Mux(NEW_ID, port1.data.extract(pos, width), port2.data.extract(pos, width), port2.en[pos])); + new_en = module->Or(NEW_ID, port1.en[pos], port2.en[pos]); + } + for (int k = pos; k < epos; k++) + port1.en[k] = new_en; + pos = epos; } - - // Connect the new EN and DATA signals and remove the old write port. - - cell->setPort(ID::EN, merged_en); - cell->setPort(ID::DATA, merged_data); - - module->remove(wr_ports[last_i]); - wr_ports[last_i] = NULL; - - log(" Active bits: "); - std::vector<RTLIL::SigBit> en_bits = sigmap(cell->getPort(ID::EN)); - active_bits_on_port.push_back(std::vector<bool>(en_bits.size())); - for (int k = int(en_bits.size())-1; k >= 0; k--) - log("%c", active_bits_on_port[i][k] ? '1' : '0'); - log("\n"); + changed = true; + port2.removed = true; } - - last_port_by_addr[addr] = i; } - // Clean up `wr_ports': remove all NULL entries - - std::vector<RTLIL::Cell*> wr_ports_with_nulls; - wr_ports_with_nulls.swap(wr_ports); + if (changed) + mem.emit(); - for (auto cell : wr_ports_with_nulls) - if (cell != NULL) - wr_ports.push_back(cell); + return changed; } @@ -486,178 +279,174 @@ struct MemoryShareWorker // Consolidate write ports using sat-based resource sharing // -------------------------------------------------------- - void consolidate_wr_using_sat(std::string memid, std::vector<RTLIL::Cell*> &wr_ports) + void consolidate_wr_using_sat(Mem &mem) { - if (wr_ports.size() <= 1) + if (GetSize(mem.wr_ports) <= 1) return; - ezSatPtr ez; - SatGen satgen(ez.get(), &modwalker.sigmap); + // Get a list of ports that have any chance of being mergeable. - // find list of considered ports and port pairs + pool<int> eligible_ports; - std::set<int> considered_ports; - std::set<int> considered_port_pairs; - - for (int i = 0; i < int(wr_ports.size()); i++) { - std::vector<RTLIL::SigBit> bits = modwalker.sigmap(wr_ports[i]->getPort(ID::EN)); + for (int i = 0; i < GetSize(mem.wr_ports); i++) { + auto &port = mem.wr_ports[i]; + std::vector<RTLIL::SigBit> bits = modwalker.sigmap(port.en); for (auto bit : bits) if (bit == RTLIL::State::S1) goto port_is_always_active; - if (modwalker.has_drivers(bits)) - considered_ports.insert(i); + eligible_ports.insert(i); port_is_always_active:; } - log("Consolidating write ports of memory %s.%s using sat-based resource sharing:\n", log_id(module), log_id(memid)); + if (eligible_ports.size() <= 1) + return; + + log("Consolidating write ports of memory %s.%s using sat-based resource sharing:\n", log_id(module), log_id(mem.memid)); - bool cache_clk_enable = false; - bool cache_clk_polarity = false; - RTLIL::SigSpec cache_clk; + // Group eligible ports by clock domain and width. - for (int i = 0; i < int(wr_ports.size()); i++) + pool<int> checked_ports; + std::vector<std::vector<int>> groups; + for (int i = 0; i < GetSize(mem.wr_ports); i++) { - RTLIL::Cell *cell = wr_ports.at(i); + auto &port1 = mem.wr_ports[i]; + if (!eligible_ports.count(i)) + continue; + if (checked_ports.count(i)) + continue; - if (cell->parameters.at(ID::CLK_ENABLE).as_bool() != cache_clk_enable || - (cache_clk_enable && (sigmap(cell->getPort(ID::CLK)) != cache_clk || - cell->parameters.at(ID::CLK_POLARITY).as_bool() != cache_clk_polarity))) + std::vector<int> group; + group.push_back(i); + + for (int j = i + 1; j < GetSize(mem.wr_ports); j++) { - cache_clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool(); - cache_clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool(); - cache_clk = sigmap(cell->getPort(ID::CLK)); + auto &port2 = mem.wr_ports[j]; + if (!eligible_ports.count(j)) + continue; + if (checked_ports.count(j)) + continue; + if (port1.clk_enable != port2.clk_enable) + continue; + if (port1.clk_enable) { + if (port1.clk != port2.clk) + continue; + if (port1.clk_polarity != port2.clk_polarity) + continue; + } + if (port1.wide_log2 != port2.wide_log2) + continue; + group.push_back(j); } - else if (i > 0 && considered_ports.count(i-1) && considered_ports.count(i)) - considered_port_pairs.insert(i); - - if (cache_clk_enable) - log(" Port %d (%s) on %s %s: %s\n", i, log_id(cell), - cache_clk_polarity ? "posedge" : "negedge", log_signal(cache_clk), - considered_ports.count(i) ? "considered" : "not considered"); - else - log(" Port %d (%s) unclocked: %s\n", i, log_id(cell), - considered_ports.count(i) ? "considered" : "not considered"); - } - if (considered_port_pairs.size() < 1) { - log(" No two subsequent ports in same clock domain considered -> nothing to consolidate.\n"); - return; - } - - // create SAT representation of common input cone of all considered EN signals + for (auto j : group) + checked_ports.insert(j); - pool<Wire*> one_hot_wires; - std::set<RTLIL::Cell*> sat_cells; - std::set<RTLIL::SigBit> bits_queue; - std::map<int, int> port_to_sat_variable; + if (group.size() <= 1) + continue; - for (int i = 0; i < int(wr_ports.size()); i++) - if (considered_port_pairs.count(i) || considered_port_pairs.count(i+1)) - { - RTLIL::SigSpec sig = modwalker.sigmap(wr_ports[i]->getPort(ID::EN)); - port_to_sat_variable[i] = ez->expression(ez->OpOr, satgen.importSigSpec(sig)); + groups.push_back(group); + } - std::vector<RTLIL::SigBit> bits = sig; - bits_queue.insert(bits.begin(), bits.end()); + bool changed = false; + for (auto &group : groups) { + auto &some_port = mem.wr_ports[group[0]]; + string ports; + for (auto idx : group) { + if (idx != group[0]) + ports += ", "; + ports += std::to_string(idx); + } + if (!some_port.clk_enable) { + log(" Checking unclocked group, width %d: ports %s.\n", mem.width << some_port.wide_log2, ports.c_str()); + } else { + log(" Checking group clocked with %sedge %s, width %d: ports %s.\n", some_port.clk_polarity ? "pos" : "neg", log_signal(some_port.clk), mem.width << some_port.wide_log2, ports.c_str()); } - while (!bits_queue.empty()) - { - for (auto bit : bits_queue) - if (bit.wire && bit.wire->get_bool_attribute(ID::onehot)) - one_hot_wires.insert(bit.wire); - - pool<ModWalker::PortBit> portbits; - modwalker.get_drivers(portbits, bits_queue); - bits_queue.clear(); - - for (auto &pbit : portbits) - if (sat_cells.count(pbit.cell) == 0 && cone_ct.cell_known(pbit.cell->type)) { - pool<RTLIL::SigBit> &cell_inputs = modwalker.cell_inputs[pbit.cell]; - bits_queue.insert(cell_inputs.begin(), cell_inputs.end()); - sat_cells.insert(pbit.cell); - } - } + // Okay, time to actually run the SAT solver. - for (auto wire : one_hot_wires) { - log(" Adding one-hot constraint for wire %s.\n", log_id(wire)); - vector<int> ez_wire_bits = satgen.importSigSpec(wire); - for (int i : ez_wire_bits) - for (int j : ez_wire_bits) - if (i != j) ez->assume(ez->NOT(i), j); - } + QuickConeSat qcsat(modwalker); - log(" Common input cone for all EN signals: %d cells.\n", int(sat_cells.size())); + // create SAT representation of common input cone of all considered EN signals - for (auto cell : sat_cells) - satgen.importCell(cell); + dict<int, int> port_to_sat_variable; - log(" Size of unconstrained SAT problem: %d variables, %d clauses\n", ez->numCnfVariables(), ez->numCnfClauses()); + for (auto idx : group) + port_to_sat_variable[idx] = qcsat.ez->expression(qcsat.ez->OpOr, qcsat.importSig(mem.wr_ports[idx].en)); - // merge subsequent ports if possible + qcsat.prepare(); - for (int i = 0; i < int(wr_ports.size()); i++) - { - if (!considered_port_pairs.count(i)) - continue; + log(" Common input cone for all EN signals: %d cells.\n", GetSize(qcsat.imported_cells)); - if (ez->solve(port_to_sat_variable.at(i-1), port_to_sat_variable.at(i))) { - log(" According to SAT solver sharing of port %d with port %d is not possible.\n", i-1, i); - continue; - } + log(" Size of unconstrained SAT problem: %d variables, %d clauses\n", qcsat.ez->numCnfVariables(), qcsat.ez->numCnfClauses()); + + // now try merging the ports. + + for (int ii = 0; ii < GetSize(group); ii++) { + int idx1 = group[ii]; + auto &port1 = mem.wr_ports[idx1]; + if (port1.removed) + continue; + for (int jj = ii + 1; jj < GetSize(group); jj++) { + int idx2 = group[jj]; + auto &port2 = mem.wr_ports[idx2]; + if (port2.removed) + continue; - log(" Merging port %d into port %d.\n", i-1, i); - port_to_sat_variable.at(i) = ez->OR(port_to_sat_variable.at(i-1), port_to_sat_variable.at(i)); + if (qcsat.ez->solve(port_to_sat_variable.at(idx1), port_to_sat_variable.at(idx2))) { + log(" According to SAT solver sharing of port %d with port %d is not possible.\n", idx1, idx2); + continue; + } - RTLIL::SigSpec last_addr = wr_ports[i-1]->getPort(ID::ADDR); - RTLIL::SigSpec last_data = wr_ports[i-1]->getPort(ID::DATA); - std::vector<RTLIL::SigBit> last_en = modwalker.sigmap(wr_ports[i-1]->getPort(ID::EN)); + log(" Merging port %d into port %d.\n", idx2, idx1); + mem.prepare_wr_merge(idx1, idx2, &initvals); + port_to_sat_variable.at(idx1) = qcsat.ez->OR(port_to_sat_variable.at(idx1), port_to_sat_variable.at(idx2)); - RTLIL::SigSpec this_addr = wr_ports[i]->getPort(ID::ADDR); - RTLIL::SigSpec this_data = wr_ports[i]->getPort(ID::DATA); - std::vector<RTLIL::SigBit> this_en = modwalker.sigmap(wr_ports[i]->getPort(ID::EN)); + RTLIL::SigSpec last_addr = port1.addr; + RTLIL::SigSpec last_data = port1.data; + std::vector<RTLIL::SigBit> last_en = modwalker.sigmap(port1.en); - RTLIL::SigBit this_en_active = module->ReduceOr(NEW_ID, this_en); + RTLIL::SigSpec this_addr = port2.addr; + RTLIL::SigSpec this_data = port2.data; + std::vector<RTLIL::SigBit> this_en = modwalker.sigmap(port2.en); - if (GetSize(last_addr) < GetSize(this_addr)) - last_addr.extend_u0(GetSize(this_addr)); - else - this_addr.extend_u0(GetSize(last_addr)); + RTLIL::SigBit this_en_active = module->ReduceOr(NEW_ID, this_en); - wr_ports[i]->setParam(ID::ABITS, GetSize(this_addr)); - wr_ports[i]->setPort(ID::ADDR, module->Mux(NEW_ID, last_addr, this_addr, this_en_active)); - wr_ports[i]->setPort(ID::DATA, module->Mux(NEW_ID, last_data, this_data, this_en_active)); + if (GetSize(last_addr) < GetSize(this_addr)) + last_addr.extend_u0(GetSize(this_addr)); + else + this_addr.extend_u0(GetSize(last_addr)); - std::map<std::pair<RTLIL::SigBit, RTLIL::SigBit>, int> groups_en; - RTLIL::SigSpec grouped_last_en, grouped_this_en, en; - RTLIL::Wire *grouped_en = module->addWire(NEW_ID, 0); + SigSpec new_addr = module->Mux(NEW_ID, last_addr.extract_end(port1.wide_log2), this_addr.extract_end(port1.wide_log2), this_en_active); - for (int j = 0; j < int(this_en.size()); j++) { - std::pair<RTLIL::SigBit, RTLIL::SigBit> key(last_en[j], this_en[j]); - if (!groups_en.count(key)) { - grouped_last_en.append(last_en[j]); - grouped_this_en.append(this_en[j]); - groups_en[key] = grouped_en->width; - grouped_en->width++; - } - en.append(RTLIL::SigSpec(grouped_en, groups_en[key])); - } + port1.addr = SigSpec({new_addr, port1.addr.extract(0, port1.wide_log2)}); + port1.data = module->Mux(NEW_ID, last_data, this_data, this_en_active); - module->addMux(NEW_ID, grouped_last_en, grouped_this_en, this_en_active, grouped_en); - wr_ports[i]->setPort(ID::EN, en); + std::map<std::pair<RTLIL::SigBit, RTLIL::SigBit>, int> groups_en; + RTLIL::SigSpec grouped_last_en, grouped_this_en, en; + RTLIL::Wire *grouped_en = module->addWire(NEW_ID, 0); - module->remove(wr_ports[i-1]); - wr_ports[i-1] = NULL; - } + for (int j = 0; j < int(this_en.size()); j++) { + std::pair<RTLIL::SigBit, RTLIL::SigBit> key(last_en[j], this_en[j]); + if (!groups_en.count(key)) { + grouped_last_en.append(last_en[j]); + grouped_this_en.append(this_en[j]); + groups_en[key] = grouped_en->width; + grouped_en->width++; + } + en.append(RTLIL::SigSpec(grouped_en, groups_en[key])); + } - // Clean up `wr_ports': remove all NULL entries + module->addMux(NEW_ID, grouped_last_en, grouped_this_en, this_en_active, grouped_en); + port1.en = en; - std::vector<RTLIL::Cell*> wr_ports_with_nulls; - wr_ports_with_nulls.swap(wr_ports); + port2.removed = true; + changed = true; + } + } + } - for (auto cell : wr_ports_with_nulls) - if (cell != NULL) - wr_ports.push_back(cell); + if (changed) + mem.emit(); } @@ -665,26 +454,19 @@ struct MemoryShareWorker // Setup and run // ------------- - MemoryShareWorker(RTLIL::Design *design) : design(design), modwalker(design) {} + MemoryShareWorker(RTLIL::Design *design, bool flag_widen, bool flag_sat) : design(design), modwalker(design), flag_widen(flag_widen), flag_sat(flag_sat) {} void operator()(RTLIL::Module* module) { - std::map<std::string, std::pair<std::vector<RTLIL::Cell*>, std::vector<RTLIL::Cell*>>> memindex; + std::vector<Mem> memories = Mem::get_selected_memories(module); this->module = module; sigmap.set(module); - sig_to_mux.clear(); - conditions_logic_cache.clear(); + initvals.set(&sigmap, module); sigmap_xmux = sigmap; for (auto cell : module->cells()) { - if (cell->type == ID($memrd)) - memindex[cell->parameters.at(ID::MEMID).decode_string()].first.push_back(cell); - - if (cell->type == ID($memwr)) - memindex[cell->parameters.at(ID::MEMID).decode_string()].second.push_back(cell); - if (cell->type == ID($mux)) { RTLIL::SigSpec sig_a = sigmap_xmux(cell->getPort(ID::A)); @@ -695,40 +477,20 @@ struct MemoryShareWorker else if (sig_b.is_fully_undef()) sigmap_xmux.add(cell->getPort(ID::Y), sig_a); } - - if (cell->type.in(ID($mux), ID($pmux))) - { - std::vector<RTLIL::SigBit> sig_y = sigmap(cell->getPort(ID::Y)); - for (int i = 0; i < int(sig_y.size()); i++) - sig_to_mux[sig_y[i]] = std::pair<RTLIL::Cell*, int>(cell, i); - } } - for (auto &it : memindex) { - std::sort(it.second.first.begin(), it.second.first.end(), memcells_cmp); - std::sort(it.second.second.begin(), it.second.second.end(), memcells_cmp); - translate_rd_feedback_to_en(it.first, it.second.first, it.second.second); - consolidate_wr_by_addr(it.first, it.second.second); + for (auto &mem : memories) { + while (consolidate_rd_by_addr(mem)); + while (consolidate_wr_by_addr(mem)); } - cone_ct.setup_internals(); - cone_ct.cell_types.erase(ID($mul)); - cone_ct.cell_types.erase(ID($mod)); - cone_ct.cell_types.erase(ID($div)); - cone_ct.cell_types.erase(ID($modfloor)); - cone_ct.cell_types.erase(ID($divfloor)); - cone_ct.cell_types.erase(ID($pow)); - cone_ct.cell_types.erase(ID($shl)); - cone_ct.cell_types.erase(ID($shr)); - cone_ct.cell_types.erase(ID($sshl)); - cone_ct.cell_types.erase(ID($sshr)); - cone_ct.cell_types.erase(ID($shift)); - cone_ct.cell_types.erase(ID($shiftx)); - - modwalker.setup(module, &cone_ct); - - for (auto &it : memindex) - consolidate_wr_using_sat(it.first, it.second.second); + if (!flag_sat) + return; + + modwalker.setup(module); + + for (auto &mem : memories) + consolidate_wr_using_sat(mem); } }; @@ -738,22 +500,22 @@ struct MemorySharePass : public Pass { { // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| log("\n"); - log(" memory_share [selection]\n"); + log(" memory_share [-nosat] [-nowiden] [selection]\n"); log("\n"); log("This pass merges share-able memory ports into single memory ports.\n"); log("\n"); log("The following methods are used to consolidate the number of memory ports:\n"); log("\n"); - log(" - When write ports are connected to async read ports accessing the same\n"); - log(" address, then this feedback path is converted to a write port with\n"); - log(" byte/part enable signals.\n"); - log("\n"); log(" - When multiple write ports access the same address then this is converted\n"); log(" to a single write port with a more complex data and/or enable logic path.\n"); log("\n"); + log(" - When multiple read or write ports access adjacent aligned addresses, they are\n"); + log(" merged to a single wide read or write port. This transformation can be\n"); + log(" disabled with the \"-nowiden\" option.\n"); + log("\n"); log(" - When multiple write ports are never accessed at the same time (a SAT\n"); log(" solver is used to determine this), then the ports are merged into a single\n"); - log(" write port.\n"); + log(" write port. This transformation can be disabled with the \"-nosat\" option.\n"); log("\n"); log("Note that in addition to the algorithms implemented in this pass, the $memrd\n"); log("and $memwr cells are also subject to generic resource sharing passes (and other\n"); @@ -761,9 +523,26 @@ struct MemorySharePass : public Pass { log("\n"); } void execute(std::vector<std::string> args, RTLIL::Design *design) override { + bool flag_widen = true; + bool flag_sat = true; log_header(design, "Executing MEMORY_SHARE pass (consolidating $memrd/$memwr cells).\n"); + size_t argidx; + for (argidx = 1; argidx < args.size(); argidx++) + { + if (args[argidx] == "-nosat") + { + flag_sat = false; + continue; + } + if (args[argidx] == "-nowiden") + { + flag_widen = false; + continue; + } + break; + } extra_args(args, 1, design); - MemoryShareWorker msw(design); + MemoryShareWorker msw(design, flag_widen, flag_sat); for (auto module : design->selected_modules()) msw(module); diff --git a/passes/memory/memory_unpack.cc b/passes/memory/memory_unpack.cc index d04d4ba7a..422d6fe15 100644 --- a/passes/memory/memory_unpack.cc +++ b/passes/memory/memory_unpack.cc @@ -1,7 +1,7 @@ /* * yosys -- Yosys Open SYnthesis Suite * - * Copyright (C) 2012 Clifford Wolf <clifford@clifford.at> + * Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com> * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,114 +17,12 @@ * */ -#include "kernel/register.h" -#include "kernel/log.h" -#include <sstream> -#include <algorithm> -#include <stdlib.h> +#include "kernel/yosys.h" +#include "kernel/mem.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN -void handle_memory(RTLIL::Module *module, RTLIL::Cell *memory) -{ - log("Creating $memrd and $memwr for memory `%s' in module `%s':\n", - memory->name.c_str(), module->name.c_str()); - - RTLIL::IdString mem_name = RTLIL::escape_id(memory->parameters.at(ID::MEMID).decode_string()); - - while (module->memories.count(mem_name) != 0) - mem_name = mem_name.str() + stringf("_%d", autoidx++); - - RTLIL::Memory *mem = new RTLIL::Memory; - mem->name = mem_name; - mem->width = memory->parameters.at(ID::WIDTH).as_int(); - mem->start_offset = memory->parameters.at(ID::OFFSET).as_int(); - mem->size = memory->parameters.at(ID::SIZE).as_int(); - module->memories[mem_name] = mem; - - int abits = memory->parameters.at(ID::ABITS).as_int(); - int num_rd_ports = memory->parameters.at(ID::RD_PORTS).as_int(); - int num_wr_ports = memory->parameters.at(ID::WR_PORTS).as_int(); - - for (int i = 0; i < num_rd_ports; i++) - { - RTLIL::Cell *cell = module->addCell(NEW_ID, ID($memrd)); - cell->parameters[ID::MEMID] = mem_name.str(); - cell->parameters[ID::ABITS] = memory->parameters.at(ID::ABITS); - cell->parameters[ID::WIDTH] = memory->parameters.at(ID::WIDTH); - cell->parameters[ID::CLK_ENABLE] = RTLIL::SigSpec(memory->parameters.at(ID::RD_CLK_ENABLE)).extract(i, 1).as_const(); - cell->parameters[ID::CLK_POLARITY] = RTLIL::SigSpec(memory->parameters.at(ID::RD_CLK_POLARITY)).extract(i, 1).as_const(); - cell->parameters[ID::TRANSPARENT] = RTLIL::SigSpec(memory->parameters.at(ID::RD_TRANSPARENT)).extract(i, 1).as_const(); - cell->setPort(ID::CLK, memory->getPort(ID::RD_CLK).extract(i, 1)); - cell->setPort(ID::EN, memory->getPort(ID::RD_EN).extract(i, 1)); - cell->setPort(ID::ADDR, memory->getPort(ID::RD_ADDR).extract(i*abits, abits)); - cell->setPort(ID::DATA, memory->getPort(ID::RD_DATA).extract(i*mem->width, mem->width)); - } - - for (int i = 0; i < num_wr_ports; i++) - { - RTLIL::Cell *cell = module->addCell(NEW_ID, ID($memwr)); - cell->parameters[ID::MEMID] = mem_name.str(); - cell->parameters[ID::ABITS] = memory->parameters.at(ID::ABITS); - cell->parameters[ID::WIDTH] = memory->parameters.at(ID::WIDTH); - cell->parameters[ID::CLK_ENABLE] = RTLIL::SigSpec(memory->parameters.at(ID::WR_CLK_ENABLE)).extract(i, 1).as_const(); - cell->parameters[ID::CLK_POLARITY] = RTLIL::SigSpec(memory->parameters.at(ID::WR_CLK_POLARITY)).extract(i, 1).as_const(); - cell->parameters[ID::PRIORITY] = i; - cell->setPort(ID::CLK, memory->getPort(ID::WR_CLK).extract(i, 1)); - cell->setPort(ID::EN, memory->getPort(ID::WR_EN).extract(i*mem->width, mem->width)); - cell->setPort(ID::ADDR, memory->getPort(ID::WR_ADDR).extract(i*abits, abits)); - cell->setPort(ID::DATA, memory->getPort(ID::WR_DATA).extract(i*mem->width, mem->width)); - } - - Const initval = memory->parameters.at(ID::INIT); - RTLIL::Cell *last_init_cell = nullptr; - SigSpec last_init_data; - int last_init_addr=0; - - for (int i = 0; i < GetSize(initval) && i/mem->width < (1 << abits); i += mem->width) { - Const val = initval.extract(i, mem->width, State::Sx); - for (auto bit : val.bits) - if (bit != State::Sx) - goto found_non_undef_initval; - continue; - found_non_undef_initval: - if (last_init_cell && last_init_addr+1 == i/mem->width) { - last_init_cell->parameters[ID::WORDS] = last_init_cell->parameters[ID::WORDS].as_int() + 1; - last_init_data.append(val); - last_init_addr++; - } else { - if (last_init_cell) - last_init_cell->setPort(ID::DATA, last_init_data); - RTLIL::Cell *cell = module->addCell(NEW_ID, ID($meminit)); - cell->parameters[ID::MEMID] = mem_name.str(); - cell->parameters[ID::ABITS] = memory->parameters.at(ID::ABITS); - cell->parameters[ID::WIDTH] = memory->parameters.at(ID::WIDTH); - cell->parameters[ID::WORDS] = 1; - cell->parameters[ID::PRIORITY] = i/mem->width; - cell->setPort(ID::ADDR, SigSpec(i/mem->width, abits)); - last_init_cell = cell; - last_init_addr = i/mem->width; - last_init_data = val; - } - } - - if (last_init_cell) - last_init_cell->setPort(ID::DATA, last_init_data); - - module->remove(memory); -} - -void handle_module(RTLIL::Design *design, RTLIL::Module *module) -{ - std::vector<RTLIL::IdString> memcells; - for (auto cell : module->cells()) - if (cell->type == ID($mem) && design->selected(module, cell)) - memcells.push_back(cell->name); - for (auto &it : memcells) - handle_memory(module, module->cell(it)); -} - struct MemoryUnpackPass : public Pass { MemoryUnpackPass() : Pass("memory_unpack", "unpack multi-port memory cells") { } void help() override @@ -140,8 +38,14 @@ struct MemoryUnpackPass : public Pass { void execute(std::vector<std::string> args, RTLIL::Design *design) override { log_header(design, "Executing MEMORY_UNPACK pass (generating $memrd/$memwr cells form $mem cells).\n"); extra_args(args, 1, design); - for (auto module : design->selected_modules()) - handle_module(design, module); + for (auto module : design->selected_modules()) { + for (auto &mem : Mem::get_selected_memories(module)) { + if (mem.packed) { + mem.packed = false; + mem.emit(); + } + } + } } } MemoryUnpackPass; |