Further work on the Montgomery modular multiplier. Added the third

"rectangular" stage of the multiplication process, i.e. computation of how many
copies of the modulus N to add to the intermediate product AB to zeroize the
lower half: M = Q * N.

1 files changed, 290 insertions, 72 deletions

diff --git a/bench/tb_square.v b/bench/tb_square.v
index 23831db..d35a5cc 100644
--- a/bench/tb_square.v
+++ b/bench/tb_square.v
@@ -41,6 +41,8 @@ module tb_square;
     reg [17:0] AB[0:63];
     reg [17:0] N_COEFF[0:32];
     reg [17:0] Q[0:32];
+    reg [17:0] N[0:31];
+    reg [17:0] M[0:64];
 
 
     //
@@ -103,6 +105,33 @@ module tb_square;
         Q[28] = 18'h0bf39; Q[29] = 18'h0929d; Q[30] = 18'h05273; Q[31] = 18'h0c30a;
         Q[32] = 18'h0eef3;
         //
+        N[ 0] = 18'h03ad9; N[ 1] = 18'h046b4; N[ 2] = 18'h0e181; N[ 3] = 18'h0fac7;
+        N[ 4] = 18'h0be72; N[ 5] = 18'h029ab; N[ 6] = 18'h07e51; N[ 7] = 18'h037a8;
+        N[ 8] = 18'h0880c; N[ 9] = 18'h05a7d; N[10] = 18'h043c2; N[11] = 18'h038c9;
+        N[12] = 18'h01275; N[13] = 18'h0aa0d; N[14] = 18'h0c0c1; N[15] = 18'h0d035;
+        N[16] = 18'h04082; N[17] = 18'h0543c; N[18] = 18'h0dcb0; N[19] = 18'h0497c;
+        N[20] = 18'h0b12c; N[21] = 18'h013d4; N[22] = 18'h0b80a; N[23] = 18'h051cf;
+        N[24] = 18'h0286c; N[25] = 18'h0b600; N[26] = 18'h0d838; N[27] = 18'h0af4b;
+        N[28] = 18'h08274; N[29] = 18'h06a07; N[30] = 18'h0beea; N[31] = 18'h0f000;
+        //
+        M[ 0] = 18'h041b2; M[ 1] = 18'h00128; M[ 2] = 18'h06b69; M[ 3] = 18'h08e7e;
+        M[ 4] = 18'h0118c; M[ 5] = 18'h0b96d; M[ 6] = 18'h0ebe5; M[ 7] = 18'h0f873;
+        M[ 8] = 18'h0cf14; M[ 9] = 18'h0de83; M[10] = 18'h09690; M[11] = 18'h05e9a;
+        M[12] = 18'h048ac; M[13] = 18'h0b506; M[14] = 18'h01283; M[15] = 18'h08631;
+        M[16] = 18'h0179c; M[17] = 18'h06820; M[18] = 18'h0867b; M[19] = 18'h0b750;
+        M[20] = 18'h0e680; M[21] = 18'h0df95; M[22] = 18'h0d818; M[23] = 18'h0b4c5;
+        M[24] = 18'h0cced; M[25] = 18'h0c4a9; M[26] = 18'h0bb78; M[27] = 18'h04295;
+        M[28] = 18'h0b1b4; M[29] = 18'h09635; M[30] = 18'h0066b; M[31] = 18'h022b1;
+        M[32] = 18'h04fdb; M[33] = 18'h0efc8; M[34] = 18'h00a14; M[35] = 18'h04bef;
+        M[36] = 18'h006a1; M[37] = 18'h0f1a6; M[38] = 18'h0fc40; M[39] = 18'h0adb5;
+        M[40] = 18'h06e8f; M[41] = 18'h02c60; M[42] = 18'h083e1; M[43] = 18'h0f862;
+        M[44] = 18'h0da61; M[45] = 18'h0dd3d; M[46] = 18'h03381; M[47] = 18'h09db0;
+        M[48] = 18'h05454; M[49] = 18'h07525; M[50] = 18'h0d9c7; M[51] = 18'h0a361;
+        M[52] = 18'h049e0; M[53] = 18'h0a671; M[54] = 18'h0242e; M[55] = 18'h07cb2;
+        M[56] = 18'h02021; M[57] = 18'h0bde1; M[58] = 18'h025aa; M[59] = 18'h0c615;
+        M[60] = 18'h05645; M[61] = 18'h03b46; M[62] = 18'h065d6; M[63] = 18'h0390d;
+        M[64] = 18'h0e005;
+        //  
     end
     
 
@@ -134,6 +163,12 @@ module tb_square;
     reg [ 7:0] tb_slim_bram_xy_addr;
     reg [17:0] tb_slim_bram_x_din;
     reg [17:0] tb_slim_bram_y_din;
+    
+    reg        mgr_slim_bram_xy_ena = 1'b0;
+    reg [ 1:0] mgr_slim_bram_xy_bank;
+    reg [ 7:0] mgr_slim_bram_xy_addr;
+    reg [17:0] mgr_slim_bram_x_din;
+    reg [17:0] mgr_slim_bram_y_din;
 
     reg         mac_slim_bram_xy_ena = 1'b0;
     reg         mac_slim_bram_xy_reg_ena = 1'b0;
@@ -195,10 +230,10 @@ module tb_square;
     ip_bram_18k slim_bram_x
     (
         .clka   (clk),
-        .ena    (tb_slim_bram_xy_ena),
-        .wea    (tb_slim_bram_xy_ena),
-        .addra  ({tb_slim_bram_xy_bank, tb_slim_bram_xy_addr}),
-        .dina   (tb_slim_bram_x_din),
+        .ena    (mgr_slim_bram_xy_ena),
+        .wea    (mgr_slim_bram_xy_ena),
+        .addra  ({mgr_slim_bram_xy_bank, mgr_slim_bram_xy_addr}),
+        .dina   (mgr_slim_bram_x_din),
     
         .clkb   (clk),
         .enb    (mac_slim_bram_xy_ena),
@@ -210,10 +245,10 @@ module tb_square;
     ip_bram_18k slim_bram_y
     (
         .clka   (clk),
-        .ena    (tb_slim_bram_xy_ena),
-        .wea    (tb_slim_bram_xy_ena),
-        .addra  ({tb_slim_bram_xy_bank, tb_slim_bram_xy_addr}),
-        .dina   (tb_slim_bram_y_din),
+        .ena    (mgr_slim_bram_xy_ena),
+        .wea    (mgr_slim_bram_xy_ena),
+        .addra  ({mgr_slim_bram_xy_bank, mgr_slim_bram_xy_addr}),
+        .dina   (mgr_slim_bram_y_din),
     
         .clkb   (clk),
         .enb    (mac_slim_bram_xy_ena),
@@ -266,14 +301,23 @@ module tb_square;
             wait_clock_tick;
         end
         for (i=32; i<33; i=i+1) begin
-            tb_slim_bram_xy_bank = BANK_SLIM_N_COEFF_EXT;
-            tb_slim_bram_xy_addr = 0;
+            tb_slim_bram_xy_bank = BANK_SLIM_EXT;
+            tb_slim_bram_xy_addr = 0;   // !
             tb_slim_bram_x_din = N_COEFF[i];
             tb_slim_bram_y_din = N_COEFF[i];
             
             wait_clock_tick;
         end
 
+        for (i=0; i<32; i=i+1) begin
+            tb_fat_bram_xy_bank = BANK_FAT_N;
+            tb_fat_bram_xy_addr = i[7:0];
+            tb_fat_bram_x_din = N[i];
+            tb_fat_bram_y_din = N[i];
+            
+            wait_clock_tick;
+        end
+
         tb_fat_bram_xy_ena = 1'b0;        
         tb_slim_bram_xy_ena = 1'b0;
         
@@ -299,6 +343,7 @@ module tb_square;
             
         verify_ab;
         verify_q;
+        verify_m;
 
     end
 
@@ -418,25 +463,23 @@ module tb_square;
 
     wire mult_square_addr_almost_done_comb;
     reg  mult_square_addr_almost_done_flop;
-        
-    //wire mult_square_addr_surely_done_comb;
     reg  mult_square_addr_surely_done_flop; 
 
-    reg  mult_triangle_addr_almost_done_comb;
-    reg  mult_triangle_addr_almost_done_flop;
-        
-    //wire mult_triangle_addr_surely_done_comb;
+    wire  mult_triangle_addr_almost_done_comb;
+    reg  mult_triangle_addr_almost_done_flop;        
     reg  mult_triangle_addr_surely_done_flop;
     reg  mult_triangle_addr_tardy_done_flop;
+
+    wire  mult_rectangle_addr_almost_done_comb;
+    reg  mult_rectangle_addr_almost_done_flop;        
+    reg  mult_rectangle_addr_surely_done_flop;
+    reg  mult_rectangle_addr_tardy_done_flop;
+
     
     assign mult_square_addr_almost_done_comb = mac_slim_bram_xy_addr == index_last_minus1;
+    assign mult_triangle_addr_almost_done_comb = (mac_slim_bram_xy_addr[2:0] == index_last_minus1[2:0]) && (mac_slim_bram_xy_addr[7:3] == col_index);
+    assign mult_rectangle_addr_almost_done_comb = mac_slim_bram_xy_addr == index_last_minus1;
 
-    always @*
-        //
-        //if (!col_is_last)
-            mult_triangle_addr_almost_done_comb = (mac_slim_bram_xy_addr[2:0] == index_last_minus1[2:0]) && (mac_slim_bram_xy_addr[7:3] == col_index);
-        //else
-            //mult_triangle_addr_almost_done_comb = (mac_slim_bram_xy_addr[2:0] == index_last[2:0]) && (mac_slim_bram_xy_addr[7:3] == col_index);
             
     
     
@@ -482,6 +525,29 @@ module tb_square;
         //
     end
         
+        
+     always @(posedge clk)
+        //
+        case (fsm_state)
+        
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:
+                mult_rectangle_addr_almost_done_flop <= mult_rectangle_addr_almost_done_comb;
+                //{mult_triangle_addr_surely_done_flop, mult_triangle_addr_almost_done_flop} <= 
+                //{mult_triangle_addr_surely_done_comb, mult_triangle_addr_almost_done_comb};
+                
+            default:
+                mult_rectangle_addr_almost_done_flop <= 1'b0;
+                //{mult_triangle_addr_surely_done_flop, mult_triangle_addr_almost_done_flop} <= 2'b00;
+            
+        endcase
+
+    always @(posedge clk) begin
+        //
+        mult_rectangle_addr_surely_done_flop <= mult_rectangle_addr_almost_done_flop;
+        mult_rectangle_addr_tardy_done_flop  <= mult_rectangle_addr_surely_done_flop;
+        //
+    end
 
 
     //
@@ -489,6 +555,7 @@ module tb_square;
     //
     wire [FSM_STATE_WIDTH-1:0] fsm_state_after_mult_square;
     wire [FSM_STATE_WIDTH-1:0] fsm_state_after_mult_triangle;
+    wire [FSM_STATE_WIDTH-1:0] fsm_state_after_mult_rectangle;
     
     
     always @(posedge clk)
@@ -510,6 +577,14 @@ module tb_square;
             FSM_STATE_MULT_TRIANGLE_COL_N_BUSY: mac_slim_bram_xy_addr <= mult_triangle_addr_almost_done_flop || (col_is_last && mult_triangle_addr_surely_done_flop) ?
                 8'd0 :  mac_slim_bram_xy_addr + 1'b1;
             //
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT: mac_slim_bram_xy_addr <= 8'd0;
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY: mac_slim_bram_xy_addr <= mult_rectangle_addr_almost_done_flop || mult_rectangle_addr_surely_done_flop ?
+                8'd1 :  mac_slim_bram_xy_addr + 1'b1;            
+            //
             default:                            mac_slim_bram_xy_addr <= 8'dX;
         endcase
 
@@ -543,7 +618,14 @@ module tb_square;
                 FSM_STATE_MULT_TRIANGLE_COL_0_TRIG,
                 FSM_STATE_MULT_TRIANGLE_COL_N_TRIG,
                 FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
-                FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   mac_fat_bram_xy_addr[j] <= mac_fat_bram_xy_addr_next(mac_fat_bram_xy_addr[j], index_last);                
+                FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   mac_fat_bram_xy_addr[j] <= mac_fat_bram_xy_addr_next(mac_fat_bram_xy_addr[j], index_last);
+                //
+                FSM_STATE_MULT_RECTANGLE_COL_0_INIT:   mac_fat_bram_xy_addr[j] <= {5'd0, fat_bram_offset_rom[j]};
+                FSM_STATE_MULT_RECTANGLE_COL_N_INIT:   mac_fat_bram_xy_addr[j] <= {col_index_next1, fat_bram_offset_rom[j]};
+                FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+                FSM_STATE_MULT_RECTANGLE_COL_N_TRIG,
+                FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+                FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:   mac_fat_bram_xy_addr[j] <= mac_fat_bram_xy_addr_next(mac_fat_bram_xy_addr[j], index_last);
                 //
                 default:                            mac_fat_bram_xy_addr[j] <= 8'dX;
             endcase
@@ -564,7 +646,14 @@ module tb_square;
             FSM_STATE_MULT_TRIANGLE_COL_0_TRIG,
             FSM_STATE_MULT_TRIANGLE_COL_N_TRIG,
             FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
-            FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   mac_fat_bram_xy_addr[4] <= mac_fat_bram_xy_addr_next(mac_fat_bram_xy_addr[4], index_last);                
+            FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   mac_fat_bram_xy_addr[4] <= mac_fat_bram_xy_addr_next(mac_fat_bram_xy_addr[4], index_last);
+            //
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT:   mac_fat_bram_xy_addr[4] <= {5'd0, 3'd1};
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT:   mac_fat_bram_xy_addr[4] <= {5'd0, 3'd1};
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:   mac_fat_bram_xy_addr[4] <= mac_fat_bram_xy_addr_next(mac_fat_bram_xy_addr[4], index_last);
             //
             default:                            mac_fat_bram_xy_addr[4] <= 8'dX;
         endcase
@@ -574,19 +663,30 @@ module tb_square;
     always @(posedge clk)
         //
         case (fsm_state_next)
+            //
             FSM_STATE_MULT_SQUARE_COL_0_INIT,
             FSM_STATE_MULT_SQUARE_COL_N_INIT,
             FSM_STATE_MULT_SQUARE_COL_0_TRIG,
             FSM_STATE_MULT_SQUARE_COL_N_TRIG,
             FSM_STATE_MULT_SQUARE_COL_0_BUSY,
             FSM_STATE_MULT_SQUARE_COL_N_BUSY:   mac_slim_bram_xy_bank <= BANK_SLIM_T1T2;
+            //
             FSM_STATE_MULT_TRIANGLE_COL_0_INIT,
             FSM_STATE_MULT_TRIANGLE_COL_N_INIT,
             FSM_STATE_MULT_TRIANGLE_COL_0_TRIG,
             FSM_STATE_MULT_TRIANGLE_COL_N_TRIG,
             FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
             FSM_STATE_MULT_TRIANGLE_COL_N_BUSY: mac_slim_bram_xy_bank <= col_is_last && (mult_triangle_addr_almost_done_flop || mult_triangle_addr_surely_done_flop) ?
-                BANK_SLIM_N_COEFF_EXT : BANK_SLIM_N_COEFF;  
+                BANK_SLIM_EXT : BANK_SLIM_N_COEFF;
+            //
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY: mac_slim_bram_xy_bank <= mult_rectangle_addr_almost_done_flop || mult_rectangle_addr_surely_done_flop ?
+                BANK_SLIM_EXT : BANK_SLIM_Q;            
+            //
             default:                            mac_slim_bram_xy_bank <= 2'bXX;
         endcase
 
@@ -605,6 +705,12 @@ module tb_square;
             FSM_STATE_MULT_TRIANGLE_COL_N_TRIG: {mac_fat_bram_xy_bank_aux, mac_fat_bram_xy_bank} <= {BANK_FAT_ABH, BANK_FAT_ABL};
             FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
             FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   {mac_fat_bram_xy_bank_aux, mac_fat_bram_xy_bank} <= {2{BANK_FAT_ABL}};
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG,    
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:   {mac_fat_bram_xy_bank_aux, mac_fat_bram_xy_bank} <= {2{BANK_FAT_N}};            
             default:                            {mac_fat_bram_xy_bank_aux, mac_fat_bram_xy_bank} <= {2{3'bXXX}};
         endcase
 
@@ -625,6 +731,12 @@ module tb_square;
             FSM_STATE_MULT_TRIANGLE_COL_N_TRIG:   mac_slim_bram_xy_ena <= 1'b1;
             FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
             FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   mac_slim_bram_xy_ena <= !col_is_last ? ~mult_triangle_addr_almost_done_flop : ~mult_triangle_addr_surely_done_flop; 
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG:   mac_slim_bram_xy_ena <= 1'b1;
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:   mac_slim_bram_xy_ena <= ~mult_rectangle_addr_surely_done_flop;
             default:                              mac_slim_bram_xy_ena <= 1'b0;
         endcase
 
@@ -642,7 +754,13 @@ module tb_square;
             FSM_STATE_MULT_TRIANGLE_COL_0_TRIG,
             FSM_STATE_MULT_TRIANGLE_COL_N_TRIG,
             FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
-            FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:   mac_fat_bram_xy_ena <= 1'b1;
+            FSM_STATE_MULT_TRIANGLE_COL_N_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:   mac_fat_bram_xy_ena <= 1'b1;
             default:                            mac_fat_bram_xy_ena <= 1'b0;
         endcase
 
@@ -654,12 +772,30 @@ module tb_square;
     always @(posedge clk)
         //
         mac_fat_bram_xy_reg_ena <= mac_fat_bram_xy_ena;
-            
+          
+    reg ladder_mode = 1'b0; // 0 = X:T1*T2, Y:T2*T2
+                            // 1 = X:T1*T2, Y:T2*T1
+          
 
+    reg dsp_swap_xy;
+    
+    always @(posedge clk)
+        //
+        case (fsm_state)
+            FSM_STATE_MULT_SQUARE_COL_0_TRIG:   dsp_swap_xy <= 1'b1;
+            FSM_STATE_MULT_TRIANGLE_COL_0_TRIG: dsp_swap_xy <= 1'b0;
+        endcase
+  
     always @(posedge clk)
         //
-        if (mac_slim_bram_xy_reg_ena_dly)
-            {dsp_y_b, dsp_x_b} <= {mac_slim_bram_x_dout[16:0], mac_slim_bram_y_dout[16:0]};
+        if (mac_slim_bram_xy_reg_ena_dly) begin // rewrite
+            if (!dsp_swap_xy)
+                {dsp_y_b, dsp_x_b} <= {mac_slim_bram_y_dout[16:0], mac_slim_bram_x_dout[16:0]};
+            else begin
+                if (!ladder_mode) {dsp_y_b, dsp_x_b} <= {mac_slim_bram_x_dout[16:0], mac_slim_bram_y_dout[16:0]};
+                else              {dsp_y_b, dsp_x_b} <= {mac_slim_bram_y_dout[16:0], mac_slim_bram_x_dout[16:0]};
+            end
+        end
         else
             {dsp_y_b, dsp_x_b} <= {2{{17{1'bX}}}};
 
@@ -711,7 +847,8 @@ module tb_square;
         case (fsm_state_next)
             //
             FSM_STATE_MULT_SQUARE_COL_0_INIT,
-            FSM_STATE_MULT_TRIANGLE_COL_0_INIT: begin
+            FSM_STATE_MULT_TRIANGLE_COL_0_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT: begin
                 col_index       <= 5'd0;
                 col_index_last  <= index_last[7:3];
                 col_index_next1 <= 5'd1;
@@ -721,7 +858,8 @@ module tb_square;
             end
             //
             FSM_STATE_MULT_SQUARE_COL_N_INIT,
-            FSM_STATE_MULT_TRIANGLE_COL_N_INIT: begin
+            FSM_STATE_MULT_TRIANGLE_COL_N_INIT,
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT: begin
                 col_index <= col_index_next1;
                 col_is_last <= col_index_next1 == col_index_last;
                 col_index_next1 <= col_index_next1 == col_index_last ? 5'd0 : col_index_next1 + 5'd1;   
@@ -730,8 +868,9 @@ module tb_square;
             //
         endcase
     
-    assign  fsm_state_after_mult_square   = col_is_last ? FSM_STATE_MULT_SQUARE_HOLDOFF   : FSM_STATE_MULT_SQUARE_COL_N_INIT;
-    assign  fsm_state_after_mult_triangle = col_is_last ? FSM_STATE_MULT_TRIANGLE_HOLDOFF : FSM_STATE_MULT_TRIANGLE_COL_N_INIT;
+    assign fsm_state_after_mult_square    = col_is_last ? FSM_STATE_MULT_SQUARE_HOLDOFF   : FSM_STATE_MULT_SQUARE_COL_N_INIT;
+    assign fsm_state_after_mult_triangle  = col_is_last ? FSM_STATE_MULT_TRIANGLE_HOLDOFF : FSM_STATE_MULT_TRIANGLE_COL_N_INIT;
+    assign fsm_state_after_mult_rectangle = col_is_last ? FSM_STATE_MULT_RECTANGLE_HOLDOFF : FSM_STATE_MULT_RECTANGLE_COL_N_INIT;
     
     always @(posedge clk)
         //
@@ -741,9 +880,13 @@ module tb_square;
             FSM_STATE_MULT_SQUARE_COL_0_BUSY,
             FSM_STATE_MULT_SQUARE_COL_N_BUSY:       dsp_xy_mode_z_adv4 <= calc_mac_mode_z_square(col_index_prev, mac_slim_bram_xy_addr_dly);
             FSM_STATE_MULT_TRIANGLE_COL_0_TRIG,
-            FSM_STATE_MULT_TRIANGLE_COL_N_TRIG:     dsp_xy_mode_z_adv4 <= {9{1'b0}};
+            FSM_STATE_MULT_TRIANGLE_COL_N_TRIG:     dsp_xy_mode_z_adv4 <= {9{1'b0}};    // so easy
             FSM_STATE_MULT_TRIANGLE_COL_0_BUSY,
             FSM_STATE_MULT_TRIANGLE_COL_N_BUSY:     dsp_xy_mode_z_adv4 <= {9{1'b1}};
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG,
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG:     dsp_xy_mode_z_adv4 <= {9{1'b0}};    // so easy
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY,
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY:     dsp_xy_mode_z_adv4 <= calc_mac_mode_z_rectangle(col_index_prev, mac_slim_bram_xy_addr_dly);
             default:                                dsp_xy_mode_z_adv4 <= {9{1'b1}};
         endcase
 
@@ -774,26 +917,26 @@ module tb_square;
                 calc_mac_mode_z_square = {1'b1, {NUM_MULTS{1'b1}}};
         end
     endfunction
-    /*
-    function  [NUM_MULTS:0] calc_mac_mode_z_triangle;
+    
+    function  [NUM_MULTS:0] calc_mac_mode_z_rectangle;
         input [        4:0] col_index_value;
         input [        7:0] mac_slim_bram_xy_addr_value;
         begin
             if (mac_slim_bram_xy_addr_value[7:3] == col_index_value)
                 case (mac_slim_bram_xy_addr_value[2:0])
-                    3'b000: calc_mac_mode_z_square = {1'b1, 8'b11111110};
-                    3'b001: calc_mac_mode_z_square = {1'b1, 8'b11111101};
-                    3'b010: calc_mac_mode_z_square = {1'b1, 8'b11111011};
-                    3'b011: calc_mac_mode_z_square = {1'b1, 8'b11110111};
-                    3'b100: calc_mac_mode_z_square = {1'b1, 8'b11101111};
-                    3'b101: calc_mac_mode_z_square = {1'b1, 8'b11011111};
-                    3'b110: calc_mac_mode_z_square = {1'b1, 8'b10111111};
-                    3'b111: calc_mac_mode_z_square = {1'b1, 8'b01111111};
+                    3'b000: calc_mac_mode_z_rectangle = {1'b1, 8'b11111110};
+                    3'b001: calc_mac_mode_z_rectangle = {1'b1, 8'b11111101};
+                    3'b010: calc_mac_mode_z_rectangle = {1'b1, 8'b11111011};
+                    3'b011: calc_mac_mode_z_rectangle = {1'b1, 8'b11110111};
+                    3'b100: calc_mac_mode_z_rectangle = {1'b1, 8'b11101111};
+                    3'b101: calc_mac_mode_z_rectangle = {1'b1, 8'b11011111};
+                    3'b110: calc_mac_mode_z_rectangle = {1'b1, 8'b10111111};
+                    3'b111: calc_mac_mode_z_rectangle = {1'b1, 8'b01111111};
                 endcase
             else
-                calc_mac_mode_z_square = {1'b1, {NUM_MULTS{1'b1}}};
+                calc_mac_mode_z_rectangle = {1'b1, {NUM_MULTS{1'b1}}};
         end
-    endfunction*/
+    endfunction
 
     reg recomb_x_ena = 1'b0;
     reg recomb_y_ena = 1'b0;
@@ -810,44 +953,66 @@ module tb_square;
     wire [17:0] recomb_fat_bram_x_dout;
     wire [17:0] recomb_fat_bram_y_dout;
     wire        recomb_fat_bram_xy_dout_valid;
+    wire [ 2:0] recomb_slim_bram_xy_bank;
+    wire [ 7:0] recomb_slim_bram_xy_addr;
+    wire [17:0] recomb_slim_bram_x_dout;
+    wire [17:0] recomb_slim_bram_y_dout;
+    wire        recomb_slim_bram_xy_dout_valid;
     wire        recomb_rdy;
     
     modexpng_part_recombinator recomb
     (
-        .clk                        (clk),
-        .rdy                        (recomb_rdy),
-        .fsm_state_next             (fsm_state_next),
-        .index_last                 (index_last),
-        .dsp_x_ce_p                 (dsp_x_ce_p),
-        .dsp_y_ce_p                 (dsp_y_ce_p),
-        .ena_x                      (recomb_x_ena),
-        .ena_y                      (recomb_y_ena),
-        .dsp_x_p                    (dsp_x_p),
-        .dsp_y_p                    (dsp_y_p),
-        .col_index                  (col_index),
-        .col_index_last             (col_index_last),
-        .slim_bram_xy_addr          (mac_slim_bram_xy_addr),
-        .slim_bram_xy_bank          (mac_slim_bram_xy_bank),
-        .fat_bram_xy_bank           (recomb_fat_bram_xy_bank),
-        .fat_bram_xy_addr           (recomb_fat_bram_xy_addr),
-        .fat_bram_x_dout            (recomb_fat_bram_x_dout),
-        .fat_bram_y_dout            (recomb_fat_bram_y_dout),
-        .fat_bram_xy_dout_valid     (recomb_fat_bram_xy_dout_valid)
+        .clk                            (clk),
+        .rdy                            (recomb_rdy),
+        .fsm_state_next                 (fsm_state_next),
+        .index_last                     (index_last),
+        .dsp_x_ce_p                     (dsp_x_ce_p),
+        .dsp_y_ce_p                     (dsp_y_ce_p),
+        .ena_x                          (recomb_x_ena),
+        .ena_y                          (recomb_y_ena),
+        .dsp_x_p                        (dsp_x_p),
+        .dsp_y_p                        (dsp_y_p),
+        .col_index                      (col_index),
+        .col_index_last                 (col_index_last),
+        .slim_bram_xy_addr              (mac_slim_bram_xy_addr),
+        .slim_bram_xy_bank              (mac_slim_bram_xy_bank),
+        .rcmb_fat_bram_xy_bank          (recomb_fat_bram_xy_bank),
+        .rcmb_fat_bram_xy_addr          (recomb_fat_bram_xy_addr),
+        .rcmb_fat_bram_x_dout           (recomb_fat_bram_x_dout),
+        .rcmb_fat_bram_y_dout           (recomb_fat_bram_y_dout),
+        .rcmb_fat_bram_xy_dout_valid    (recomb_fat_bram_xy_dout_valid),
+        .rcmb_slim_bram_xy_bank         (recomb_slim_bram_xy_bank),
+        .rcmb_slim_bram_xy_addr         (recomb_slim_bram_xy_addr),
+        .rcmb_slim_bram_x_dout          (recomb_slim_bram_x_dout),
+        .rcmb_slim_bram_y_dout          (recomb_slim_bram_y_dout),
+        .rcmb_slim_bram_xy_dout_valid   (recomb_slim_bram_xy_dout_valid)
     );
     
     reg [17:0] AB_READ[0:63];
     reg [17:0] Q_READ[0:32];
+    reg [17:0] M_READ[0:64];
     
-    always @(posedge clk)
+    always @(posedge clk) begin
         //
         if (recomb_fat_bram_xy_dout_valid)
             //
             case (recomb_fat_bram_xy_bank)
-                3'd1: AB_READ[     (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
-                3'd2: AB_READ[32 + (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
-                3'd3: Q_READ [     (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
-                3'd4: Q_READ [32 + (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
+                BANK_FAT_ABL: AB_READ[recomb_fat_bram_xy_addr % 32] <= recomb_fat_bram_x_dout;
+                BANK_FAT_ABH: AB_READ[32 + (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
+                BANK_FAT_ML:  M_READ[recomb_fat_bram_xy_addr % 32] <= recomb_fat_bram_x_dout;
+                BANK_FAT_MH:  M_READ[32 + (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
+                BANK_FAT_EXT: M_READ[64 + (recomb_fat_bram_xy_addr % 32)] <= recomb_fat_bram_x_dout;
             endcase
+            //
+        if (recomb_slim_bram_xy_dout_valid)
+            //
+            case (recomb_slim_bram_xy_bank)
+                BANK_SLIM_Q: Q_READ[recomb_slim_bram_xy_addr] <= recomb_slim_bram_x_dout;
+                BANK_SLIM_EXT: if (recomb_slim_bram_xy_addr == 8'd1)
+                             Q_READ[32] <= recomb_slim_bram_x_dout;
+            endcase
+            //
+    end
             
 
     always @(posedge clk)
@@ -873,6 +1038,29 @@ module tb_square;
         end
 
 
+    always @(posedge clk)
+        //
+        if (tb_slim_bram_xy_ena) begin
+            mgr_slim_bram_xy_ena  <= 1'b1;
+            mgr_slim_bram_xy_bank <= tb_slim_bram_xy_bank;
+            mgr_slim_bram_xy_addr <= tb_slim_bram_xy_addr;
+            mgr_slim_bram_x_din   <= tb_slim_bram_x_din;
+            mgr_slim_bram_y_din   <= tb_slim_bram_y_din;
+        end else if (recomb_slim_bram_xy_dout_valid) begin
+            mgr_slim_bram_xy_ena  <= 1'b1;
+            mgr_slim_bram_xy_bank <= recomb_slim_bram_xy_bank;
+            mgr_slim_bram_xy_addr <= recomb_slim_bram_xy_addr;
+            mgr_slim_bram_x_din   <= recomb_slim_bram_x_dout;
+            mgr_slim_bram_y_din   <= recomb_slim_bram_y_dout;
+        end else begin
+            mgr_slim_bram_xy_ena  <= 1'b0;
+            mgr_slim_bram_xy_bank <= 3'bXXX;
+            mgr_slim_bram_xy_addr <= 8'hXX;
+            mgr_slim_bram_x_din   <= {18{1'bX}};
+            mgr_slim_bram_y_din   <= {18{1'bX}};
+        end
+
+
     task verify_ab;
         reg verify_ab_ok;
         begin
@@ -911,9 +1099,29 @@ module tb_square;
     endtask
 
 
+    task verify_m;
+        reg verify_m_ok;
+        begin
+            verify_m_ok = 1;
+            for (i=0; i<65; i=i+1)
+                if (M_READ[i] === M[i])
+                    $display("M / M_READ [%02d] = 0x%05x / 0x%05x", i, M[i], M_READ[i]);
+                else begin
+                    $display("M / M_READ [%02d] = 0x%05x / 0x%05x <???>", i, M[i], M_READ[i]);
+                    verify_m_ok = 0;
+                end
+            if (verify_m_ok)
+                $display("M is OK.");
+            else
+                $display("M is WRONG!");
+        end
+    endtask
+
+
     wire mult_square_addr_done = mult_square_addr_surely_done_flop;
-    
     wire mult_triangle_addr_done = !col_is_last ? mult_triangle_addr_surely_done_flop : mult_triangle_addr_tardy_done_flop;
+    wire mult_rectangle_addr_done = mult_rectangle_addr_tardy_done_flop;
+    
 
     always @* begin
         //
@@ -940,7 +1148,17 @@ module tb_square;
             FSM_STATE_MULT_TRIANGLE_COL_N_TRIG: fsm_state_next =                         FSM_STATE_MULT_TRIANGLE_COL_N_BUSY ;
             FSM_STATE_MULT_TRIANGLE_COL_N_BUSY: fsm_state_next = mult_triangle_addr_done ? fsm_state_after_mult_triangle : FSM_STATE_MULT_TRIANGLE_COL_N_BUSY;
             
-            FSM_STATE_MULT_TRIANGLE_HOLDOFF:    fsm_state_next =                         FSM_STATE_MULT_TRIANGLE_HOLDOFF;//recomb_rdy ? FSM_STATE_IDLE : FSM_STATE_MULT_SQUARE_HOLDOFF;
+            FSM_STATE_MULT_TRIANGLE_HOLDOFF:    fsm_state_next =                         recomb_rdy ? FSM_STATE_MULT_RECTANGLE_COL_0_INIT : FSM_STATE_MULT_TRIANGLE_HOLDOFF;
+
+            FSM_STATE_MULT_RECTANGLE_COL_0_INIT: fsm_state_next =                         FSM_STATE_MULT_RECTANGLE_COL_0_TRIG ;
+            FSM_STATE_MULT_RECTANGLE_COL_0_TRIG: fsm_state_next =                         FSM_STATE_MULT_RECTANGLE_COL_0_BUSY ;
+            FSM_STATE_MULT_RECTANGLE_COL_0_BUSY: fsm_state_next = mult_rectangle_addr_done ? FSM_STATE_MULT_RECTANGLE_COL_N_INIT : FSM_STATE_MULT_RECTANGLE_COL_0_BUSY;     
+            
+            FSM_STATE_MULT_RECTANGLE_COL_N_INIT: fsm_state_next =                         FSM_STATE_MULT_RECTANGLE_COL_N_TRIG ;
+            FSM_STATE_MULT_RECTANGLE_COL_N_TRIG: fsm_state_next =                         FSM_STATE_MULT_RECTANGLE_COL_N_BUSY ;
+            FSM_STATE_MULT_RECTANGLE_COL_N_BUSY: fsm_state_next = mult_rectangle_addr_done ? fsm_state_after_mult_rectangle : FSM_STATE_MULT_RECTANGLE_COL_N_BUSY;
+            
+            FSM_STATE_MULT_RECTANGLE_HOLDOFF:    fsm_state_next =                         recomb_rdy ? FSM_STATE_STOP : FSM_STATE_MULT_RECTANGLE_HOLDOFF;
             
             default:                          fsm_state_next =                         FSM_STATE_IDLE                   ;