<turbo-stream action="append" target="posts_list"><template>    <div class="postbit" id="140765" data-post-id="140765">
  <section>
    <div class="post-wrap">


					<div class="post-header">
		        <div class="user-avatar">
		          <img alt="CharlesO" src="https://forum.elixirforum.com/user_avatar/forum.elixirforum.com/CharlesO/120/34571_2.png" width="120" height="120" />
		        </div>
					
						<div class="user-details">
		          <div class="user-name">
		            <h3>
                  CharlesO
                    <span class="op-star" title="Thread Starter">
                      <img alt="OP" class="op-star-icon" src="/assets/thread-icons/thread-icon-thread-starter-df91e872.png" />
                    </span>
                  </h3>
		          </div>
						
						</div>
					
					</div>

	        <div class="thread-main">
	            <div class="post-body" data-turbo="false">
								<aside class="quote no-group" data-username="NobbZ" data-post="9" data-topic="24816">
<div class="title">
<div class="quote-controls"></div>
<img alt="" width="24" height="24" src="https://forum.elixirforum.com/user_avatar/forum.elixirforum.com/nobbz/48/27235_2.png" class="avatar"> NobbZ:</div>
<blockquote>
<p>I’m not sure how to refactor though. As I currently will not try to understand the full parser, as I do not know JQL anyway…</p>
</blockquote>
</aside>
<pre data-code-wrap="elixir"><code class="lang-elixir">000007775B4C4900000136490000000049FFFFFFFF4C49000000674900000000 4900000005 # row count
	490000000049000000006A62 00000165 3136333737303030313335323832312E303130303031FE3136333737303030313335323832312E303130303031FE3030303030303030303139373339FE4E4730303130303031FE333439312E3030FE323133FE414C4C4F434154494F4EFE414C4C4F434154494F4EFEFE31FE31FE35303032FE3230313231313031FEFE465431323330363030303636FEFE3230313231313031FE31FE33FE465431323330363030303636FE4654FE3230313231313031FE3136333737303030313335323832312E3031FD312D32FEFEFE31FE31323131303131343430FE4E474EFE333439312E3030FEFE4445424954FEFE3230313231313031FE3230313231313031FEFEFEFEFEFEFEFEFEFEFEFEFEFE41432E312E54522E4E474E2E353030322E312E353032302E343930302E4E472E4E472E313030302D312E312E2E2E2E2E4E4730303130303031FEFE44454641554C54FE4E4730303130303031FEFEFE3230313231313031FE31FEFE-3B3B
	490000000049000000006A62 0000014D 3136393233303030303936333935342E303130303032FE3136393233303030303936333935342E303130303032FE4E474E31313236303030303130303031FE4E4730303130303031FE2D3439343534392E3031FE323133FEFEFEFEFE31FE3131323630FE3230313430343239FEFE465431343131393030303036FEFE3230313430343239FE31FE37FE465431343131393030303036FE4654FE3230313430343239FE3136393233303030303936333935342E3031FD312D32FEFEFE31FE31343035303131373435FE4E474EFE2D3439343534392E3031FEFE4445424954FEFE3230313430343239FE3230313430343239FEFEFEFEFEFEFEFEFEFEFEFEFEFE41432E312E54522E4E474E2E31313236302E372E2E2E2E2E313030302D312E2E2E2E2E2E4E4730303130303031FEFE44454641554C54FE4E4730303130303031FEFEFE3230313430343239FE31FEFE-3B3B
	490000000049000000006A62 00000178 3136393834303030303534313938352E303230303032FE3136393834303030303534313938352E303230303032FE3030303030303030303231303234FE4E4730303130303031FE2D36393032322E3636FE323133FE434F4C4C454354494F4EFE434F4C4C454354494F4EFEFE31303030FE31FE35303033FE3230313430363330FEFE465431343138323030313432FEFE3230313430373031FE31FE33FE465431343138323030313432FE4654FE3230313430373031FE3136393834303030303534313938352E3032FD312D32FEFEFE31FE31343037303131313339FE4E474EFE2D36393032322E3636FEFE435245444954FEFE3230313430363330FE3230313430363330FEFEFEFEFEFEFEFEFE3230313430373031FEFEFEFEFE41432E312E54522E4E474E2E353030332E372E353032302E343930302E4E472E4E472E313030302D312E313030302E2E2E2E2E4E4730303130303031FEFE44454641554C54FE4E4730303130303031FEFEFE3230313430373031FE31FEFE-3B3B
	490000000049000000006A62 0000015C 3136393834303030303634363137372E303030303031FE3136393834303030303634363137372E303030303031FE3030303030303030303139383434FE4E4730303130303032FE3230353038302E3838FE323133FEFEFEFE32303030FE31FE31323036FE3230313430373031FEFE465431343138323030333438FEFE3230313430373031FE31FE37FE465431343138323030333438FE4654FE3230313430373031FE3136393834303030303634363137372E3030FD31FD31FEFEFE31FE31343037303131323439FE4E474EFE3230353038302E3838FEFE435245444954FEFE3230313430373031FE3230313430373031FEFEFEFEFEFEFEFEFEFEFEFEFEFE41432E312E54522E4E474E2E313230362E372E353032302E343930302E4E472E4E472E323030302D312E323030302E2E2E2E2E4E4730303130303032FEFE44454641554C54FE4E4730303130303032FEFEFE3230313430373031FE31FEFE-3B3B
	490000000049000000006A62 00000173 3136393834303030323435323030362E303230303031FE3136393834303030323435323030362E303230303031FE3030303030303030303139393431FE4E4730303130303031FE3130383336382E3030FE323133FE434C45415245442044455441494C53FE434C45415245442044455441494C53FEFE31303030FE31FE31303130FE3230313430373031FEFE465431343138323030343633FEFE3230313430373031FE31FE33FE465431343138323030343633FE4654FE3230313430373031FE3136393834303030323435323030362E3032FD312D32FEFEFE31FE31343037303131343236FE4E474EFE3130383336382E3030FEFE435245444954FEFE3230313430373031FE3230313430373031FEFEFEFEFEFEFEFEFEFEFEFEFEFE41432E312E54522E4E474E2E313031302E372E353032302E343930302E4E472E4E472E313030302D312E313030302E2E2E2E2E4E4730303130303031FEFEFE4E4730303130303031FEFEFE3230313430373031FE31FEFE-3B3B
3B3B5D
</code></pre>
<p>JQL is not so tricky. You can see the core repeating pattern above:</p>
<pre data-code-wrap="elixir"><code class="lang-elixir">data-body header row-count
row-header - row-length - row data - row terminator
...
...
data-body terminator
</code></pre> 
	            </div>

	            <div class="base-line">
	                <div class="thread-counters">
	                    <span class="thread-count count-likes js-likers-trigger" title="Likes" data-post-id="140765" data-batch-url="/posts/batch_likers">
                        0
                      </span>
                      <!-- <span class="thread-count js-solved-indicator" title="Marked as solution"></span> -->
	                </div>
	                <div class="go-to-post">
	                  <a title="Go to post" alt="Go to post" href="https://forum.elixirforum.com/t/parse-pattern-match-large-binary-data/24816/13">Post #12</a>
	                </div>
	            </div>
              <div id="likers-container-140765" 
                   class="likers-container"
                   data-first-post="false"
                   data-batch-url="/posts/batch_likers">
                   <div class="likers-placeholder" 
                     data-likers-post-id="140765"
                     data-batch-url="/posts/batch_likers">
                  <div class="post-likers"></div>
                </div>
              </div>
	        </div>
			

    </div>

    <div class="triangle-top-right type-standard-post cat-standard-post" title="Post #12"></div>
  </section>
</div>
    <div class="postbit" id="140769" data-post-id="140769">
  <section>
    <div class="post-wrap">


					<div class="post-header">
		        <div class="user-avatar">
		          <img alt="CharlesO" src="https://forum.elixirforum.com/user_avatar/forum.elixirforum.com/CharlesO/120/34571_2.png" width="120" height="120" />
		        </div>
					
						<div class="user-details">
		          <div class="user-name">
		            <h3>
                  CharlesO
                    <span class="op-star" title="Thread Starter">
                      <img alt="OP" class="op-star-icon" src="/assets/thread-icons/thread-icon-thread-starter-df91e872.png" />
                    </span>
                  </h3>
		          </div>
						
						</div>
					
					</div>

	        <div class="thread-main">
	            <div class="post-body" data-turbo="false">
								<p>Thanks for this. This is recursion 101, i know but i felt the size of each row was small enough to accumulate the parsed items left - right, as they occur in the data packet. The place i’ve really goofed is here:</p>
<pre data-code-wrap="elixir"><code class="lang-elixir"> defp _r(
         &lt;&lt;"LI", 0x136::32, "I", 0::32, "I", _::32, "LI", 0x67::32, "I", 0::32, "I", n::32, b::bits&gt;&gt;,
         []
       ) do
    v =
      for &lt;&lt;l::32, s::bytes-size(l), _::bits&gt;&gt; &lt;-
            :binary.split(b, @_00jb, [:global]),
          do: _split(s)

    {:data, n, v}
  end
</code></pre>
<p>on this line:</p>
<pre data-code-wrap="elixir"><code class="lang-elixir"> v =
      for &lt;&lt;l::32, s::bytes-size(l), _::bits&gt;&gt; &lt;-
            :binary.split(b, @_00jb, [:global]),
          do: _split(s)
</code></pre>
<p>i’m practically saying split the 850mb blob of data (2.5m+ rows), and build a list out of it … all at once in memory</p>
<p>When i run same routine for 5, 100 or even 100,000 rows it complete without issues.</p>
<p>I need a way to process and discard each row and not accumulate to lists like i’m doing, like you guys have rightly pointed out,  more so <code>NOT</code> using <code>++</code> to accumulate</p> 
	            </div>

	            <div class="base-line">
	                <div class="thread-counters">
	                    <span class="thread-count count-likes js-likers-trigger" title="Likes" data-post-id="140769" data-batch-url="/posts/batch_likers">
                        0
                      </span>
                      <!-- <span class="thread-count js-solved-indicator" title="Marked as solution"></span> -->
	                </div>
	                <div class="go-to-post">
	                  <a title="Go to post" alt="Go to post" href="https://forum.elixirforum.com/t/parse-pattern-match-large-binary-data/24816/14">Post #13</a>
	                </div>
	            </div>
              <div id="likers-container-140769" 
                   class="likers-container"
                   data-first-post="false"
                   data-batch-url="/posts/batch_likers">
                   <div class="likers-placeholder" 
                     data-likers-post-id="140769"
                     data-batch-url="/posts/batch_likers">
                  <div class="post-likers"></div>
                </div>
              </div>
	        </div>
			

    </div>

    <div class="triangle-top-right type-standard-post cat-standard-post" title="Post #13"></div>
  </section>
</div>
    <div class="postbit" id="140770" data-post-id="140770">
  <section>
    <div class="post-wrap">


					<div class="post-header">
		        <div class="user-avatar">
		          <img alt="benwilson512" src="https://forum.elixirforum.com/user_avatar/forum.elixirforum.com/benwilson512/120/1457_2.png" width="120" height="120" />
		        </div>
					
						<div class="user-details">
		          <div class="user-name">
		            <h3>
                  benwilson512
                  </h3>
		          </div>
						
			          <div class="user-title">
									<span>Author of Craft GraphQL APIs in Elixir with Absinthe</span>
			          </div>
						</div>
					
					</div>

	        <div class="thread-main">
	            <div class="post-body" data-turbo="false">
								<aside class="quote no-group" data-username="CharlesO" data-post="14" data-topic="24816">
<div class="title">
<div class="quote-controls"></div>
<img alt="" width="24" height="24" src="https://forum.elixirforum.com/user_avatar/forum.elixirforum.com/charleso/48/34571_2.png" class="avatar"> CharlesO:</div>
<blockquote>
<p>i felt the size of each row was small enough to accumulate the parsed items left - right, as they occur in the data packet.</p>
</blockquote>
</aside>
<p>You can do this, you just have to write it body recursive instead of tail recursive.</p> 
	            </div>

	            <div class="base-line">
	                <div class="thread-counters">
	                    <span class="thread-count count-likes js-likers-trigger" title="Likes" data-post-id="140770" data-batch-url="/posts/batch_likers">
                        1
                      </span>
                      <!-- <span class="thread-count js-solved-indicator" title="Marked as solution"></span> -->
	                </div>
	                <div class="go-to-post">
	                  <a title="Go to post" alt="Go to post" href="https://forum.elixirforum.com/t/parse-pattern-match-large-binary-data/24816/15">Post #14</a>
	                </div>
	            </div>
              <div id="likers-container-140770" 
                   class="likers-container"
                   data-first-post="false"
                   data-batch-url="/posts/batch_likers">
                   <div class="likers-placeholder" 
                     data-likers-post-id="140770"
                     data-batch-url="/posts/batch_likers">
                  <div class="post-likers"></div>
                </div>
              </div>
	        </div>
			

    </div>

    <div class="triangle-top-right type-standard-post cat-standard-post" title="Post #14"></div>
  </section>
</div>
    <div class="postbit" id="140807" data-post-id="140807">
  <section>
    <div class="post-wrap">


					<div class="post-header">
		        <div class="user-avatar">
		          <img alt="CharlesO" src="https://forum.elixirforum.com/user_avatar/forum.elixirforum.com/CharlesO/120/34571_2.png" width="120" height="120" />
		        </div>
					
						<div class="user-details">
		          <div class="user-name">
		            <h3>
                  CharlesO
                    <span class="op-star" title="Thread Starter">
                      <img alt="OP" class="op-star-icon" src="/assets/thread-icons/thread-icon-thread-starter-df91e872.png" />
                    </span>
                  </h3>
		          </div>
						
						</div>
					
					</div>

	        <div class="thread-main">
	            <div class="post-body" data-turbo="false">
								<p>Found a really simple way to solve this. Just read the data-header, then parse row by row:</p>
<pre data-code-wrap="elixir"><code class="lang-elixir">  def read(src, cnt \\ nil) do
    {:ok, fid} = :file.open("_cache/#{src}", [:raw, :read, :binary])
    {:h, start, n} = _p(fid, :h)

    cnt = cnt || n

    Logger.debug("Reading: #{cnt} of #{n} rows")

    Process.put(:start_pos, start)

    for i &lt;- 1..cnt do
      {:r, start, row} = _p(fid, Process.get(:start_pos), :r)
      Process.put(:start_pos, start)
      Logger.debug(inspect({i, row}, @format))
    end

    :file.close(fid)
  end

  # READER
  defp _p(fid, :h) do
    # 000007775B4C4900000136490000000049FFFFFFFF4C490000006749000000004900000005
    {:ok, &lt;&lt;n::32&gt;&gt;} = :file.pread(fid, 33, 4)
    {:h, 37, n}
  end

  defp _p(fid, start, :r) do
    # 490000000049000000006A62 00000165 3136333737303....
    {:ok, &lt;&lt;l::32&gt;&gt;} = :file.pread(fid, start + 12, 4)
    {:ok, b} = :file.pread(fid, start + 16, l)

    {:r, start + l + 18, _split(b)}
  end

  defp _split(v) do
    b = for(&lt;&lt;c &lt;- v&gt;&gt;, c in 32..126 || c in [252, 253, 254], into: "", do: &lt;&lt;c&gt;&gt;)
    b = :binary.replace(b, &lt;&lt;253&gt;&gt;, ";", [:global])
    b = :binary.replace(b, &lt;&lt;252&gt;&gt;, "^", [:global])
    :binary.split(b, &lt;&lt;254&gt;&gt;, [:global])
  end
</code></pre> 
	            </div>

	            <div class="base-line">
	                <div class="thread-counters">
	                    <span class="thread-count count-likes js-likers-trigger" title="Likes" data-post-id="140807" data-batch-url="/posts/batch_likers">
                        3
                      </span>
                      <!-- <span class="thread-count js-solved-indicator" title="Marked as solution"></span> -->
	                </div>
	                <div class="go-to-post">
	                  <a title="Go to post" alt="Go to post" href="https://forum.elixirforum.com/t/parse-pattern-match-large-binary-data/24816/16">Post #15</a>
	                </div>
	            </div>
              <div id="likers-container-140807" 
                   class="likers-container"
                   data-first-post="false"
                   data-batch-url="/posts/batch_likers">
                   <div class="likers-placeholder" 
                     data-likers-post-id="140807"
                     data-batch-url="/posts/batch_likers">
                  <div class="post-likers"></div>
                </div>
              </div>
	        </div>
			

    </div>

    <div class="triangle-top-right type-solved cat-solved" title="Marked as solution"></div>
  </section>
</div>
</template></turbo-stream><turbo-stream action="replace" target="load-more-container"><template><div id="load-more-container" class="load-more-container">
    <span class="all-loaded">— All posts loaded —</span>
</div></template></turbo-stream>