Skip to content

Instantly share code, notes, and snippets.

@leonardb

leonardb/locus-corrupt-download-test

Created January 9, 2020 22:37
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save leonardb/4d2b1755d13af1e65830b61767d18c68 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save leonardb/4d2b1755d13af1e65830b61767d18c68 to your computer and use it in GitHub Desktop.
Download ZIP
locus corrupt download test
Raw
locus-corrupt-download-test
Diff with io:format debugging
diff --git a/src/locus_http_download.erl b/src/locus_http_download.erl
index 222d6c5..39ac326 100644
--- a/src/locus_http_download.erl
+++ b/src/locus_http_download.erl
@@ -259,6 +259,7 @@ send_request(State)
Insecure = proplists:get_value(insecure, Opts, false),
Request = {URL, Headers},
+ io:format("Locus Request:~nURL: ~p~nHeaders:~n~p~n",[URL, Headers]),
BaseHTTPOpts = [{connect_timeout, ConnectTimeout}],
ExtraHTTPOpts =
case Insecure orelse http_uri:parse(URL) of
@@ -285,6 +286,7 @@ handle_httpc_message(Msg, State)
when State#state.response_headers =:= undefined ->
case Msg of
{_, stream_start, Headers} ->
+ io:format("STREAM START [1]~n", []),
CiHeaders = lists:keymap(fun string:to_lower/1, 1, Headers),
State2 = cancel_download_start_timeout(State),
State3 = schedule_idle_download_timeout(State2),
@@ -292,16 +294,19 @@ handle_httpc_message(Msg, State)
report_event({download_started, CiHeaders}, State4),
{noreply, State4};
{_, {{_,StatusCode,StatusDesc}, Headers, Body}} when StatusCode =:= 304 ->
+ io:format("STREAM 304 [1]~n", []),
CiHeaders = lists:keymap(fun string:to_lower/1, 1, Headers),
report_event({download_dismissed, {http, {StatusCode, StatusDesc}, CiHeaders, Body}}, State),
notify_owner({finished, dismissed}, State),
{stop, normal, State};
{_, {{_,StatusCode,StatusDesc}, Headers, Body}} ->
+ io:format("STREAM download start failed [1] ~p~n", [StatusCode]),
CiHeaders = lists:keymap(fun string:to_lower/1, 1, Headers),
report_event({download_failed_to_start, {http, {StatusCode, StatusDesc}, CiHeaders, Body}}, State),
notify_owner({finished, {error, {http,StatusCode,StatusDesc}}}, State),
{stop, normal, State};
{_, {error, Reason}} ->
+ io:format("DOWNLOAD ERROR [1]: ~p~n", [Reason]),
report_event({download_failed_to_start, {error, Reason}}, State),
notify_owner({finished, {error, {http,Reason}}}, State),
{stop, normal, State}
@@ -310,12 +315,14 @@ handle_httpc_message(Msg, State)
when State#state.response_body =/= undefined ->
case Msg of
{_, stream, BodyPart} ->
+ io:format("STREAM part [2]~n", []),
#state{response_body = BodyAcc} = State,
UpdatedBodyAcc = [BodyAcc, BodyPart],
State2 = State#state{ response_body = UpdatedBodyAcc },
State3 = reschedule_idle_download_timeout(State2),
{noreply, State3};
{_, stream_end, TrailingHeaders} -> % no chunked encoding
+ io:format("STREAM END [2]: ~p~n", [TrailingHeaders]),
#state{response_headers = HeadersAcc, response_body = BodyAcc} = State,
CiTrailingHeaders = lists:keymap(fun string:to_lower/1, 1, TrailingHeaders),
Headers = lists:usort(HeadersAcc ++ CiTrailingHeaders),
@@ -326,6 +333,7 @@ handle_httpc_message(Msg, State)
notify_owner({finished, {success, Success}}, State),
{stop, normal, State};
{_, {error, Reason}} ->
+ io:format("DOWNLOAD ERROR [2]: ~p~n", [Reason]),
#state{response_body = BodyAcc} = State,
BodySizeSoFar = iolist_size(BodyAcc),
report_event({download_finished, BodySizeSoFar, {error, Reason}}, State),
Console output of attempted load and crash
Locus Request:
URL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=XXXXXXXXX&suffix=tar.gz"
Headers:
[{"accept",
"application/gzip; application/x-gzip; application/x-gtar; application/x-tgz; application/x-tar; application/octet-stream"},
{"connection","close"}]
STREAM START [1]
STREAM part [2]
...
STREAM part [2]
STREAM END [2]: [{"connection","close"},
{"date","Thu, 09 Jan 2020 22:16:06 GMT"},
{"accept-ranges","bytes"},
{"server","cloudflare"},
{"content-length","28704939"},
{"content-type","application/gzip"},
{"last-modified","Mon, 06 Jan 2020 23:51:19 GMT"},
{"set-cookie",
"__cfduid=db9d9ad774f9695a1acc57841a0895b5e1578608166; expires=Sat, 08-Feb-20 22:16:06 GMT; path=/; domain=.maxmind.com; HttpOnly; SameSite=Lax"},
{"content-disposition",
"attachment; filename=GeoLite2-City_20200107.tar.gz"},
{"cf-cache-status","DYNAMIC"},
{"expect-ct",
"max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\""},
{"cf-ray","5529c612abe95336-MIA"}]
2020-01-09 22:16:11.106 UTC [error] <0.1501.0> [locus] geoip database failed to load (remote): {decode_database_from_gzip_blob,{error,data_error,[{zlib,inflateEnd_nif,[#Ref<0.97014130.2694971397.138944>],[]},{zlib,gunzip,1,[]},{locus_loader,decode_database_from_gzip_blob,2,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,589}]},{locus_loader,handle_database_fetch_success,3,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,492}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,616}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,686}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,247}]}]}}
2020-01-09 22:16:11.106 UTC [error] <0.1500.0>@smlib_sup:init:22 CRASH REPORT Process <0.1500.0> with 0 neighbours crashed with reason: no match of right hand value {error,{decode_database_from_gzip_blob,{error,data_error,[{zlib,inflateEnd_nif,[#Ref<0.97014130.2694971397.138944>],[]},{zlib,gunzip,1,[]},{locus_loader,decode_database_from_gzip_blob,2,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,589}]},{locus_loader,handle_database_fetch_success,3,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,492}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,616}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,686}]},...]}}} in smlib_sup:init/1 line 22
2020-01-09 22:16:11.107 UTC [error] <0.1498.0> CRASH REPORT Process <0.1498.0> with 0 neighbours exited with reason: {{{badmatch,{error,{decode_database_from_gzip_blob,{error,data_error,[{zlib,inflateEnd_nif,[#Ref<0.97014130.2694971397.138944>],[]},{zlib,gunzip,1,[]},{locus_loader,decode_database_from_gzip_blob,2,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,589}]},{locus_loader,handle_database_fetch_success,3,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,492}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,616}]},{gen_server,handle_msg,6,[{file,"gen_server...."},...]},...]}}}},...},...} in application_master:init/4 line 134
2020-01-09 22:16:11.108 UTC [info] <0.1217.0> Application smlib exited with reason: {{{badmatch,{error,{decode_database_from_gzip_blob,{error,data_error,[{zlib,inflateEnd_nif,[#Ref<0.97014130.2694971397.138944>],[]},{zlib,gunzip,1,[]},{locus_loader,decode_database_from_gzip_blob,2,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,589}]},{locus_loader,handle_database_fetch_success,3,[{file,"/opt/smd/_checkouts/locus/src/locus_loader.erl"},{line,492}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,616}]},{gen_server,handle_msg,6,[{file,"gen_server...."},...]},...]}}}},...},...}
{"Kernel pid terminated",application_controller,"{application_start_failure,smlib,{{{badmatch,{error,{decode_database_from_gzip_blob,{error,data_error,[{zlib,inflateEnd_nif,[#Ref<0.97014130.2694971397.138944>],[]},{zlib,gunzip,1,[]},{locus_loader,decode_database_from_gzip_blob,2,[{file,\"/opt/smd/_checkouts/locus/src/locus_loader.erl\"},{line,589}]},{locus_loader,handle_database_fetch_success,3,[{file,\"/opt/smd/_checkouts/locus/src/locus_loader.erl\"},{line,492}]},{gen_server,try_dispatch,4,[{file,\"gen_server.erl\"},{line,616}]},{gen_server,handle_msg,6,[{file,\"gen_server.erl\"},{line,686}]},{proc_lib,init_p_do_apply,3,[{file,\"proc_lib.erl\"},{line,247}]}]}}}},[{smlib_sup,init,1,[{file,\"/opt/smd/_build/default/lib/smlib/src/smlib_sup.erl\"},{line,22}]},{supervisor,init,1,[{file,\"supervisor.erl\"},{line,294}]},{gen_server,init_it,2,[{file,\"gen_server.erl\"},{line,365}]},{gen_server,init_it,6,[{file,\"gen_server.erl\"},{line,333}]},{proc_lib,init_p_do_apply,3,[{file,\"proc_lib.erl\"},{line,247}]}]},{smlib_app,start,[normal,[]]}}}"}
Kernel pid terminated (application_controller) ({application_start_failure,smlib,{{{badmatch,{error,{decode_database_from_gzip_blob,{error,data_error,[{zlib,inflateEnd_nif,[#Ref<0.97014130.2694971397.
heart: Thu Jan 9 17:16:13 2020: Erlang is crashing .. (waiting for crash dump file)
heart: Thu Jan 9 17:16:13 2020: Would reboot. Terminating.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment